training_car_ppo2_steering_only.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Connected!\n",
      "Client Ver:1 (Min Req: 1), Server Ver:1 (Min Req: 1)\n",
      "\n",
      "['WayPoint0', 'WayPoint1', 'WayPoint2', 'WayPoint3', 'WayPoint4', 'WayPoint5', 'WayPoint6', 'WayPoint7', 'WayPoint8', 'WayPoint9']\n",
      "Wrapping the env in a DummyVecEnv.\n",
      "WARNING:tensorflow:From C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\tf_util.py:191: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\tf_util.py:200: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\policies.py:116: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\input.py:25: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\tf_layers.py:103: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\ppo2\\ppo2.py:190: The name tf.summary.scalar is deprecated. Please use tf.compat.v1.summary.scalar instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\ppo2\\ppo2.py:206: The name tf.train.AdamOptimizer is deprecated. Please use tf.compat.v1.train.AdamOptimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\ppo2\\ppo2.py:242: The name tf.summary.merge_all is deprecated. Please use tf.compat.v1.summary.merge_all instead.\n",
      "\n",
      "WARNING:tensorflow:From C:\\Users\\Hoang\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\base_class.py:1143: The name tf.summary.FileWriter is deprecated. Please use tf.compat.v1.summary.FileWriter instead.\n",
      "\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0070357993 |\n",
      "| clipfrac           | 0.10546875   |\n",
      "| explained_variance | 0.0175       |\n",
      "| fps                | 16           |\n",
      "| n_updates          | 1            |\n",
      "| policy_entropy     | 1.9395046    |\n",
      "| policy_loss        | -0.017724052 |\n",
      "| serial_timesteps   | 128          |\n",
      "| time_elapsed       | 0            |\n",
      "| total_timesteps    | 128          |\n",
      "| value_loss         | 28.030107    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0120715555 |\n",
      "| clipfrac           | 0.21289062   |\n",
      "| explained_variance | 0.463        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 10           |\n",
      "| policy_entropy     | 1.8553102    |\n",
      "| policy_loss        | -0.01296609  |\n",
      "| serial_timesteps   | 1280         |\n",
      "| time_elapsed       | 43.4         |\n",
      "| total_timesteps    | 1280         |\n",
      "| value_loss         | 18.263697    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.008309725   |\n",
      "| clipfrac           | 0.12695312    |\n",
      "| explained_variance | 0.541         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 20            |\n",
      "| policy_entropy     | 1.6414537     |\n",
      "| policy_loss        | -0.0037176665 |\n",
      "| serial_timesteps   | 2560          |\n",
      "| time_elapsed       | 83.1          |\n",
      "| total_timesteps    | 2560          |\n",
      "| value_loss         | 13.952649     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.013356076   |\n",
      "| clipfrac           | 0.2109375     |\n",
      "| explained_variance | 0.927         |\n",
      "| fps                | 33            |\n",
      "| n_updates          | 30            |\n",
      "| policy_entropy     | 1.5909884     |\n",
      "| policy_loss        | 0.00061114994 |\n",
      "| serial_timesteps   | 3840          |\n",
      "| time_elapsed       | 124           |\n",
      "| total_timesteps    | 3840          |\n",
      "| value_loss         | 3.0524535     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0056444095 |\n",
      "| clipfrac           | 0.044921875  |\n",
      "| explained_variance | 0.446        |\n",
      "| fps                | 20           |\n",
      "| n_updates          | 40           |\n",
      "| policy_entropy     | 1.55562      |\n",
      "| policy_loss        | 0.002342831  |\n",
      "| serial_timesteps   | 5120         |\n",
      "| time_elapsed       | 181          |\n",
      "| total_timesteps    | 5120         |\n",
      "| value_loss         | 12.813487    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.01817731  |\n",
      "| clipfrac           | 0.3203125   |\n",
      "| explained_variance | 0.858       |\n",
      "| fps                | 20          |\n",
      "| n_updates          | 50          |\n",
      "| policy_entropy     | 1.3689756   |\n",
      "| policy_loss        | 0.018131526 |\n",
      "| serial_timesteps   | 6400        |\n",
      "| time_elapsed       | 246         |\n",
      "| total_timesteps    | 6400        |\n",
      "| value_loss         | 11.309647   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.043017242 |\n",
      "| clipfrac           | 0.4375      |\n",
      "| explained_variance | 0.98        |\n",
      "| fps                | 18          |\n",
      "| n_updates          | 60          |\n",
      "| policy_entropy     | 1.2424453   |\n",
      "| policy_loss        | 0.015092376 |\n",
      "| serial_timesteps   | 7680        |\n",
      "| time_elapsed       | 311         |\n",
      "| total_timesteps    | 7680        |\n",
      "| value_loss         | 15.111      |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.006431671  |\n",
      "| clipfrac           | 0.064453125  |\n",
      "| explained_variance | 0.267        |\n",
      "| fps                | 20           |\n",
      "| n_updates          | 70           |\n",
      "| policy_entropy     | 1.1267666    |\n",
      "| policy_loss        | 0.0010033916 |\n",
      "| serial_timesteps   | 8960         |\n",
      "| time_elapsed       | 376          |\n",
      "| total_timesteps    | 8960         |\n",
      "| value_loss         | 10.190837    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.005511258  |\n",
      "| clipfrac           | 0.0703125    |\n",
      "| explained_variance | 0.673        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 80           |\n",
      "| policy_entropy     | 1.1406997    |\n",
      "| policy_loss        | -0.014303175 |\n",
      "| serial_timesteps   | 10240        |\n",
      "| time_elapsed       | 436          |\n",
      "| total_timesteps    | 10240        |\n",
      "| value_loss         | 55.989243    |\n",
      "-------------------------------------\n",
      "-----------------------------------\n",
      "| approxkl           | 0.0694729  |\n",
      "| clipfrac           | 0.359375   |\n",
      "| explained_variance | 0.928      |\n",
      "| fps                | 16         |\n",
      "| n_updates          | 90         |\n",
      "| policy_entropy     | 0.90442574 |\n",
      "| policy_loss        | 0.01484954 |\n",
      "| serial_timesteps   | 11520      |\n",
      "| time_elapsed       | 496        |\n",
      "| total_timesteps    | 11520      |\n",
      "| value_loss         | 14.298504  |\n",
      "-----------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0049011884 |\n",
      "| clipfrac           | 0.0625       |\n",
      "| explained_variance | 0.9          |\n",
      "| fps                | 18           |\n",
      "| n_updates          | 100          |\n",
      "| policy_entropy     | 0.8896343    |\n",
      "| policy_loss        | 0.007933365  |\n",
      "| serial_timesteps   | 12800        |\n",
      "| time_elapsed       | 568          |\n",
      "| total_timesteps    | 12800        |\n",
      "| value_loss         | 6.5142245    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.019080237 |\n",
      "| clipfrac           | 0.3359375   |\n",
      "| explained_variance | 0.314       |\n",
      "| fps                | 18          |\n",
      "| n_updates          | 110         |\n",
      "| policy_entropy     | 0.97813225  |\n",
      "| policy_loss        | 0.014486106 |\n",
      "| serial_timesteps   | 14080       |\n",
      "| time_elapsed       | 647         |\n",
      "| total_timesteps    | 14080       |\n",
      "| value_loss         | 3.7873657   |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------\n",
      "| approxkl           | 0.012594398   |\n",
      "| clipfrac           | 0.19140625    |\n",
      "| explained_variance | 0.926         |\n",
      "| fps                | 17            |\n",
      "| n_updates          | 120           |\n",
      "| policy_entropy     | 0.8222779     |\n",
      "| policy_loss        | 0.00010490767 |\n",
      "| serial_timesteps   | 15360         |\n",
      "| time_elapsed       | 726           |\n",
      "| total_timesteps    | 15360         |\n",
      "| value_loss         | 20.794827     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0107728485 |\n",
      "| clipfrac           | 0.14648438   |\n",
      "| explained_variance | 0.878        |\n",
      "| fps                | 17           |\n",
      "| n_updates          | 130          |\n",
      "| policy_entropy     | 0.5754874    |\n",
      "| policy_loss        | 0.008134501  |\n",
      "| serial_timesteps   | 16640        |\n",
      "| time_elapsed       | 802          |\n",
      "| total_timesteps    | 16640        |\n",
      "| value_loss         | 10.252644    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.014349799  |\n",
      "| clipfrac           | 0.1953125    |\n",
      "| explained_variance | 0.934        |\n",
      "| fps                | 19           |\n",
      "| n_updates          | 140          |\n",
      "| policy_entropy     | 0.79000384   |\n",
      "| policy_loss        | 0.0038757953 |\n",
      "| serial_timesteps   | 17920        |\n",
      "| time_elapsed       | 876          |\n",
      "| total_timesteps    | 17920        |\n",
      "| value_loss         | 1.7273278    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.018157497   |\n",
      "| clipfrac           | 0.23242188    |\n",
      "| explained_variance | 0.951         |\n",
      "| fps                | 27            |\n",
      "| n_updates          | 150           |\n",
      "| policy_entropy     | 0.973123      |\n",
      "| policy_loss        | -0.0026730942 |\n",
      "| serial_timesteps   | 19200         |\n",
      "| time_elapsed       | 939           |\n",
      "| total_timesteps    | 19200         |\n",
      "| value_loss         | 2.2000325     |\n",
      "--------------------------------------\n",
      "-----------------------------------\n",
      "| approxkl           | 0.14052421 |\n",
      "| clipfrac           | 0.59765625 |\n",
      "| explained_variance | 0.847      |\n",
      "| fps                | 24         |\n",
      "| n_updates          | 160        |\n",
      "| policy_entropy     | 0.79247224 |\n",
      "| policy_loss        | 0.05556929 |\n",
      "| serial_timesteps   | 20480      |\n",
      "| time_elapsed       | 987        |\n",
      "| total_timesteps    | 20480      |\n",
      "| value_loss         | 10.870501  |\n",
      "-----------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.09611795   |\n",
      "| clipfrac           | 0.38671875   |\n",
      "| explained_variance | 0.964        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 170          |\n",
      "| policy_entropy     | 0.7164877    |\n",
      "| policy_loss        | -0.007200814 |\n",
      "| serial_timesteps   | 21760        |\n",
      "| time_elapsed       | 1.04e+03     |\n",
      "| total_timesteps    | 21760        |\n",
      "| value_loss         | 14.600005    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.014989536  |\n",
      "| clipfrac           | 0.19140625   |\n",
      "| explained_variance | 0.728        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 180          |\n",
      "| policy_entropy     | 0.93668276   |\n",
      "| policy_loss        | -0.029046534 |\n",
      "| serial_timesteps   | 23040        |\n",
      "| time_elapsed       | 1.08e+03     |\n",
      "| total_timesteps    | 23040        |\n",
      "| value_loss         | 11.146063    |\n",
      "-------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.018456498    |\n",
      "| clipfrac           | 0.328125       |\n",
      "| explained_variance | 0.821          |\n",
      "| fps                | 24             |\n",
      "| n_updates          | 190            |\n",
      "| policy_entropy     | 1.1760598      |\n",
      "| policy_loss        | -0.00093453843 |\n",
      "| serial_timesteps   | 24320          |\n",
      "| time_elapsed       | 1.13e+03       |\n",
      "| total_timesteps    | 24320          |\n",
      "| value_loss         | 9.93938        |\n",
      "---------------------------------------\n",
      "Eval num_timesteps=25000, episode_reward=268.64 +/- 51.68\n",
      "Episode length: 478.80 +/- 81.24\n",
      "New best mean reward!\n",
      "------------------------------------\n",
      "| approxkl           | 0.01239533  |\n",
      "| clipfrac           | 0.10546875  |\n",
      "| explained_variance | 0.984       |\n",
      "| fps                | 25          |\n",
      "| n_updates          | 200         |\n",
      "| policy_entropy     | 0.8507116   |\n",
      "| policy_loss        | 0.005268836 |\n",
      "| serial_timesteps   | 25600       |\n",
      "| time_elapsed       | 1.36e+03    |\n",
      "| total_timesteps    | 25600       |\n",
      "| value_loss         | 5.6346016   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0137942415 |\n",
      "| clipfrac           | 0.1953125    |\n",
      "| explained_variance | -4.61        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 210          |\n",
      "| policy_entropy     | 1.0495305    |\n",
      "| policy_loss        | -0.011701884 |\n",
      "| serial_timesteps   | 26880        |\n",
      "| time_elapsed       | 1.4e+03      |\n",
      "| total_timesteps    | 26880        |\n",
      "| value_loss         | 29.573063    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.009518633  |\n",
      "| clipfrac           | 0.12109375   |\n",
      "| explained_variance | 0.879        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 220          |\n",
      "| policy_entropy     | 0.96125025   |\n",
      "| policy_loss        | 0.0047616004 |\n",
      "| serial_timesteps   | 28160        |\n",
      "| time_elapsed       | 1.45e+03     |\n",
      "| total_timesteps    | 28160        |\n",
      "| value_loss         | 3.3302927    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.018503414   |\n",
      "| clipfrac           | 0.203125      |\n",
      "| explained_variance | 0.845         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 230           |\n",
      "| policy_entropy     | 1.166432      |\n",
      "| policy_loss        | -0.0015627937 |\n",
      "| serial_timesteps   | 29440         |\n",
      "| time_elapsed       | 1.5e+03       |\n",
      "| total_timesteps    | 29440         |\n",
      "| value_loss         | 8.305094      |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.014747829  |\n",
      "| clipfrac           | 0.265625     |\n",
      "| explained_variance | 0.958        |\n",
      "| fps                | 18           |\n",
      "| n_updates          | 240          |\n",
      "| policy_entropy     | 1.1029294    |\n",
      "| policy_loss        | 0.0012959924 |\n",
      "| serial_timesteps   | 30720        |\n",
      "| time_elapsed       | 1.55e+03     |\n",
      "| total_timesteps    | 30720        |\n",
      "| value_loss         | 1.3089323    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.009692923  |\n",
      "| clipfrac           | 0.1171875    |\n",
      "| explained_variance | 0.985        |\n",
      "| fps                | 15           |\n",
      "| n_updates          | 250          |\n",
      "| policy_entropy     | 1.1117902    |\n",
      "| policy_loss        | -0.007232895 |\n",
      "| serial_timesteps   | 32000        |\n",
      "| time_elapsed       | 1.62e+03     |\n",
      "| total_timesteps    | 32000        |\n",
      "| value_loss         | 3.1783965    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0046429494 |\n",
      "| clipfrac           | 0.048828125  |\n",
      "| explained_variance | 0.987        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 260          |\n",
      "| policy_entropy     | 0.7937493    |\n",
      "| policy_loss        | -0.004827561 |\n",
      "| serial_timesteps   | 33280        |\n",
      "| time_elapsed       | 1.67e+03     |\n",
      "| total_timesteps    | 33280        |\n",
      "| value_loss         | 1.829918     |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.015822504 |\n",
      "| clipfrac           | 0.13671875  |\n",
      "| explained_variance | 0.914       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 270         |\n",
      "| policy_entropy     | 0.6288611   |\n",
      "| policy_loss        | 0.003340147 |\n",
      "| serial_timesteps   | 34560       |\n",
      "| time_elapsed       | 1.72e+03    |\n",
      "| total_timesteps    | 34560       |\n",
      "| value_loss         | 1.4099448   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.009695644   |\n",
      "| clipfrac           | 0.05859375    |\n",
      "| explained_variance | 0.371         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 280           |\n",
      "| policy_entropy     | 0.6667827     |\n",
      "| policy_loss        | -0.0067781336 |\n",
      "| serial_timesteps   | 35840         |\n",
      "| time_elapsed       | 1.77e+03      |\n",
      "| total_timesteps    | 35840         |\n",
      "| value_loss         | 4.366794      |\n",
      "--------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.029371602  |\n",
      "| clipfrac           | 0.2421875    |\n",
      "| explained_variance | 0.988        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 290          |\n",
      "| policy_entropy     | 0.7518387    |\n",
      "| policy_loss        | -0.004835124 |\n",
      "| serial_timesteps   | 37120        |\n",
      "| time_elapsed       | 1.82e+03     |\n",
      "| total_timesteps    | 37120        |\n",
      "| value_loss         | 5.516002     |\n",
      "-------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.005760678    |\n",
      "| clipfrac           | 0.091796875    |\n",
      "| explained_variance | 0.936          |\n",
      "| fps                | 25             |\n",
      "| n_updates          | 300            |\n",
      "| policy_entropy     | 0.7018969      |\n",
      "| policy_loss        | -0.00051983655 |\n",
      "| serial_timesteps   | 38400          |\n",
      "| time_elapsed       | 1.87e+03       |\n",
      "| total_timesteps    | 38400          |\n",
      "| value_loss         | 6.13276        |\n",
      "---------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.017567892 |\n",
      "| clipfrac           | 0.1875      |\n",
      "| explained_variance | 0.988       |\n",
      "| fps                | 25          |\n",
      "| n_updates          | 310         |\n",
      "| policy_entropy     | 0.72199106  |\n",
      "| policy_loss        | 0.006302129 |\n",
      "| serial_timesteps   | 39680       |\n",
      "| time_elapsed       | 1.92e+03    |\n",
      "| total_timesteps    | 39680       |\n",
      "| value_loss         | 5.964909    |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.018041512   |\n",
      "| clipfrac           | 0.21679688    |\n",
      "| explained_variance | 0.912         |\n",
      "| fps                | 26            |\n",
      "| n_updates          | 320           |\n",
      "| policy_entropy     | 0.89840686    |\n",
      "| policy_loss        | 0.00052237953 |\n",
      "| serial_timesteps   | 40960         |\n",
      "| time_elapsed       | 1.97e+03      |\n",
      "| total_timesteps    | 40960         |\n",
      "| value_loss         | 1.8523891     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.008735423  |\n",
      "| clipfrac           | 0.13085938   |\n",
      "| explained_variance | 0.989        |\n",
      "| fps                | 25           |\n",
      "| n_updates          | 330          |\n",
      "| policy_entropy     | 0.67972916   |\n",
      "| policy_loss        | -0.010754796 |\n",
      "| serial_timesteps   | 42240        |\n",
      "| time_elapsed       | 2.02e+03     |\n",
      "| total_timesteps    | 42240        |\n",
      "| value_loss         | 4.7462687    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.017142292  |\n",
      "| clipfrac           | 0.31640625   |\n",
      "| explained_variance | -0.182       |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 340          |\n",
      "| policy_entropy     | 1.2319405    |\n",
      "| policy_loss        | 0.0135752885 |\n",
      "| serial_timesteps   | 43520        |\n",
      "| time_elapsed       | 2.07e+03     |\n",
      "| total_timesteps    | 43520        |\n",
      "| value_loss         | 1.4912764    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.008894213  |\n",
      "| clipfrac           | 0.15625      |\n",
      "| explained_variance | 0.985        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 350          |\n",
      "| policy_entropy     | 1.0780151    |\n",
      "| policy_loss        | -0.000785606 |\n",
      "| serial_timesteps   | 44800        |\n",
      "| time_elapsed       | 2.12e+03     |\n",
      "| total_timesteps    | 44800        |\n",
      "| value_loss         | 0.83789754   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.032588683  |\n",
      "| clipfrac           | 0.21484375   |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 360          |\n",
      "| policy_entropy     | 0.95429367   |\n",
      "| policy_loss        | 0.0076744594 |\n",
      "| serial_timesteps   | 46080        |\n",
      "| time_elapsed       | 2.17e+03     |\n",
      "| total_timesteps    | 46080        |\n",
      "| value_loss         | 2.9147344    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.03760424  |\n",
      "| clipfrac           | 0.36914062  |\n",
      "| explained_variance | 0.428       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 370         |\n",
      "| policy_entropy     | 1.0197471   |\n",
      "| policy_loss        | 0.011995687 |\n",
      "| serial_timesteps   | 47360       |\n",
      "| time_elapsed       | 2.21e+03    |\n",
      "| total_timesteps    | 47360       |\n",
      "| value_loss         | 1.5821563   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.017363692  |\n",
      "| clipfrac           | 0.16015625   |\n",
      "| explained_variance | 0.986        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 380          |\n",
      "| policy_entropy     | 0.6490261    |\n",
      "| policy_loss        | -0.007783982 |\n",
      "| serial_timesteps   | 48640        |\n",
      "| time_elapsed       | 2.26e+03     |\n",
      "| total_timesteps    | 48640        |\n",
      "| value_loss         | 1.5744135    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0031744477 |\n",
      "| clipfrac           | 0.044921875  |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 390          |\n",
      "| policy_entropy     | 0.80768263   |\n",
      "| policy_loss        | 0.0013430861 |\n",
      "| serial_timesteps   | 49920        |\n",
      "| time_elapsed       | 2.31e+03     |\n",
      "| total_timesteps    | 49920        |\n",
      "| value_loss         | 1.774298     |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=50000, episode_reward=530.20 +/- 7.69\n",
      "Episode length: 583.00 +/- 8.63\n",
      "New best mean reward!\n",
      "------------------------------------\n",
      "| approxkl           | 0.016838105 |\n",
      "| clipfrac           | 0.29101562  |\n",
      "| explained_variance | 0.987       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 400         |\n",
      "| policy_entropy     | 0.3970994   |\n",
      "| policy_loss        | 0.012838405 |\n",
      "| serial_timesteps   | 51200       |\n",
      "| time_elapsed       | 2.46e+03    |\n",
      "| total_timesteps    | 51200       |\n",
      "| value_loss         | 1.9218224   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.016741836  |\n",
      "| clipfrac           | 0.19921875   |\n",
      "| explained_variance | -1.32        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 410          |\n",
      "| policy_entropy     | 0.54482335   |\n",
      "| policy_loss        | -0.009638987 |\n",
      "| serial_timesteps   | 52480        |\n",
      "| time_elapsed       | 2.51e+03     |\n",
      "| total_timesteps    | 52480        |\n",
      "| value_loss         | 13.805226    |\n",
      "-------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.010354263    |\n",
      "| clipfrac           | 0.15234375     |\n",
      "| explained_variance | -0.212         |\n",
      "| fps                | 30             |\n",
      "| n_updates          | 420            |\n",
      "| policy_entropy     | 0.55870736     |\n",
      "| policy_loss        | -0.00039005652 |\n",
      "| serial_timesteps   | 53760          |\n",
      "| time_elapsed       | 2.56e+03       |\n",
      "| total_timesteps    | 53760          |\n",
      "| value_loss         | 8.950984       |\n",
      "---------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0077494723 |\n",
      "| clipfrac           | 0.091796875  |\n",
      "| explained_variance | 0.991        |\n",
      "| fps                | 15           |\n",
      "| n_updates          | 430          |\n",
      "| policy_entropy     | 0.8074975    |\n",
      "| policy_loss        | -0.005182757 |\n",
      "| serial_timesteps   | 55040        |\n",
      "| time_elapsed       | 2.61e+03     |\n",
      "| total_timesteps    | 55040        |\n",
      "| value_loss         | 1.4214227    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.016236858  |\n",
      "| clipfrac           | 0.17382812   |\n",
      "| explained_variance | 0.496        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 440          |\n",
      "| policy_entropy     | 0.83187777   |\n",
      "| policy_loss        | -0.008479701 |\n",
      "| serial_timesteps   | 56320        |\n",
      "| time_elapsed       | 2.66e+03     |\n",
      "| total_timesteps    | 56320        |\n",
      "| value_loss         | 12.349065    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.016644781 |\n",
      "| clipfrac           | 0.20117188  |\n",
      "| explained_variance | 0.912       |\n",
      "| fps                | 25          |\n",
      "| n_updates          | 450         |\n",
      "| policy_entropy     | 0.7741265   |\n",
      "| policy_loss        | 0.007399791 |\n",
      "| serial_timesteps   | 57600       |\n",
      "| time_elapsed       | 2.71e+03    |\n",
      "| total_timesteps    | 57600       |\n",
      "| value_loss         | 34.67832    |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------\n",
      "| approxkl           | 0.10736586 |\n",
      "| clipfrac           | 0.48242188 |\n",
      "| explained_variance | 0.61       |\n",
      "| fps                | 28         |\n",
      "| n_updates          | 460        |\n",
      "| policy_entropy     | 0.72611755 |\n",
      "| policy_loss        | 0.02536888 |\n",
      "| serial_timesteps   | 58880      |\n",
      "| time_elapsed       | 2.76e+03   |\n",
      "| total_timesteps    | 58880      |\n",
      "| value_loss         | 1.9734051  |\n",
      "-----------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.022657176   |\n",
      "| clipfrac           | 0.37695312    |\n",
      "| explained_variance | 0.921         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 470           |\n",
      "| policy_entropy     | 1.0325197     |\n",
      "| policy_loss        | -0.0017544136 |\n",
      "| serial_timesteps   | 60160         |\n",
      "| time_elapsed       | 2.81e+03      |\n",
      "| total_timesteps    | 60160         |\n",
      "| value_loss         | 33.61988      |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.008908691 |\n",
      "| clipfrac           | 0.1015625   |\n",
      "| explained_variance | 0.956       |\n",
      "| fps                | 22          |\n",
      "| n_updates          | 480         |\n",
      "| policy_entropy     | 1.1491808   |\n",
      "| policy_loss        | 0.002336001 |\n",
      "| serial_timesteps   | 61440       |\n",
      "| time_elapsed       | 2.85e+03    |\n",
      "| total_timesteps    | 61440       |\n",
      "| value_loss         | 5.202605    |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.024327572 |\n",
      "| clipfrac           | 0.296875    |\n",
      "| explained_variance | 0.791       |\n",
      "| fps                | 25          |\n",
      "| n_updates          | 490         |\n",
      "| policy_entropy     | 1.2959198   |\n",
      "| policy_loss        | 0.002626297 |\n",
      "| serial_timesteps   | 62720       |\n",
      "| time_elapsed       | 2.9e+03     |\n",
      "| total_timesteps    | 62720       |\n",
      "| value_loss         | 6.773481    |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0148166595 |\n",
      "| clipfrac           | 0.22851562   |\n",
      "| explained_variance | 0.979        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 500          |\n",
      "| policy_entropy     | 1.2241206    |\n",
      "| policy_loss        | -0.005345843 |\n",
      "| serial_timesteps   | 64000        |\n",
      "| time_elapsed       | 2.95e+03     |\n",
      "| total_timesteps    | 64000        |\n",
      "| value_loss         | 1.506458     |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0050900085  |\n",
      "| clipfrac           | 0.072265625   |\n",
      "| explained_variance | 0.997         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 510           |\n",
      "| policy_entropy     | 1.2881874     |\n",
      "| policy_loss        | -0.0074494765 |\n",
      "| serial_timesteps   | 65280         |\n",
      "| time_elapsed       | 2.99e+03      |\n",
      "| total_timesteps    | 65280         |\n",
      "| value_loss         | 1.4057392     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.048566982  |\n",
      "| clipfrac           | 0.578125     |\n",
      "| explained_variance | 0.879        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 520          |\n",
      "| policy_entropy     | 1.5758874    |\n",
      "| policy_loss        | 0.0071190437 |\n",
      "| serial_timesteps   | 66560        |\n",
      "| time_elapsed       | 3.04e+03     |\n",
      "| total_timesteps    | 66560        |\n",
      "| value_loss         | 0.80631655   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.022001883 |\n",
      "| clipfrac           | 0.30273438  |\n",
      "| explained_variance | 0.973       |\n",
      "| fps                | 23          |\n",
      "| n_updates          | 530         |\n",
      "| policy_entropy     | 1.1960875   |\n",
      "| policy_loss        | 0.012129158 |\n",
      "| serial_timesteps   | 67840       |\n",
      "| time_elapsed       | 3.09e+03    |\n",
      "| total_timesteps    | 67840       |\n",
      "| value_loss         | 2.2030556   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.01777196  |\n",
      "| clipfrac           | 0.29296875  |\n",
      "| explained_variance | 0.998       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 540         |\n",
      "| policy_entropy     | 1.1954963   |\n",
      "| policy_loss        | 0.007715225 |\n",
      "| serial_timesteps   | 69120       |\n",
      "| time_elapsed       | 3.14e+03    |\n",
      "| total_timesteps    | 69120       |\n",
      "| value_loss         | 1.0387187   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.023621902 |\n",
      "| clipfrac           | 0.34765625  |\n",
      "| explained_variance | 0.315       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 550         |\n",
      "| policy_entropy     | 0.76065195  |\n",
      "| policy_loss        | 0.02189958  |\n",
      "| serial_timesteps   | 70400       |\n",
      "| time_elapsed       | 3.18e+03    |\n",
      "| total_timesteps    | 70400       |\n",
      "| value_loss         | 1.8993104   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.016351612 |\n",
      "| clipfrac           | 0.22265625  |\n",
      "| explained_variance | 0.889       |\n",
      "| fps                | 23          |\n",
      "| n_updates          | 560         |\n",
      "| policy_entropy     | 0.797406    |\n",
      "| policy_loss        | 0.010835597 |\n",
      "| serial_timesteps   | 71680       |\n",
      "| time_elapsed       | 3.23e+03    |\n",
      "| total_timesteps    | 71680       |\n",
      "| value_loss         | 3.3515837   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.019347245  |\n",
      "| clipfrac           | 0.22460938   |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 570          |\n",
      "| policy_entropy     | 1.0521344    |\n",
      "| policy_loss        | -0.006558819 |\n",
      "| serial_timesteps   | 72960        |\n",
      "| time_elapsed       | 3.28e+03     |\n",
      "| total_timesteps    | 72960        |\n",
      "| value_loss         | 0.87908995   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.005145158  |\n",
      "| clipfrac           | 0.068359375  |\n",
      "| explained_variance | 0.747        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 580          |\n",
      "| policy_entropy     | 0.88913274   |\n",
      "| policy_loss        | -0.008054644 |\n",
      "| serial_timesteps   | 74240        |\n",
      "| time_elapsed       | 3.33e+03     |\n",
      "| total_timesteps    | 74240        |\n",
      "| value_loss         | 3.4187443    |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=75000, episode_reward=512.80 +/- 3.06\n",
      "Episode length: 587.20 +/- 1.17\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0058346945 |\n",
      "| clipfrac           | 0.048828125  |\n",
      "| explained_variance | 0.982        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 590          |\n",
      "| policy_entropy     | 1.0632693    |\n",
      "| policy_loss        | -0.012461491 |\n",
      "| serial_timesteps   | 75520        |\n",
      "| time_elapsed       | 3.48e+03     |\n",
      "| total_timesteps    | 75520        |\n",
      "| value_loss         | 2.2505622    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.012023292 |\n",
      "| clipfrac           | 0.15820312  |\n",
      "| explained_variance | 0.912       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 600         |\n",
      "| policy_entropy     | 0.7446464   |\n",
      "| policy_loss        | 0.013412939 |\n",
      "| serial_timesteps   | 76800       |\n",
      "| time_elapsed       | 3.53e+03    |\n",
      "| total_timesteps    | 76800       |\n",
      "| value_loss         | 0.7188179   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.03851772  |\n",
      "| clipfrac           | 0.40625     |\n",
      "| explained_variance | 0.319       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 610         |\n",
      "| policy_entropy     | 0.91253555  |\n",
      "| policy_loss        | 0.015082988 |\n",
      "| serial_timesteps   | 78080       |\n",
      "| time_elapsed       | 3.57e+03    |\n",
      "| total_timesteps    | 78080       |\n",
      "| value_loss         | 2.0133636   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0079207    |\n",
      "| clipfrac           | 0.09375      |\n",
      "| explained_variance | 0.982        |\n",
      "| fps                | 19           |\n",
      "| n_updates          | 620          |\n",
      "| policy_entropy     | 0.88649505   |\n",
      "| policy_loss        | -0.009463631 |\n",
      "| serial_timesteps   | 79360        |\n",
      "| time_elapsed       | 3.62e+03     |\n",
      "| total_timesteps    | 79360        |\n",
      "| value_loss         | 4.0655193    |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------\n",
      "| approxkl           | 0.024246309 |\n",
      "| clipfrac           | 0.23046875  |\n",
      "| explained_variance | 0.97        |\n",
      "| fps                | 20          |\n",
      "| n_updates          | 630         |\n",
      "| policy_entropy     | 0.5600286   |\n",
      "| policy_loss        | 0.009308452 |\n",
      "| serial_timesteps   | 80640       |\n",
      "| time_elapsed       | 3.69e+03    |\n",
      "| total_timesteps    | 80640       |\n",
      "| value_loss         | 1.0457209   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.041386284  |\n",
      "| clipfrac           | 0.30859375   |\n",
      "| explained_variance | 0.984        |\n",
      "| fps                | 17           |\n",
      "| n_updates          | 640          |\n",
      "| policy_entropy     | 0.7520357    |\n",
      "| policy_loss        | 0.0013450631 |\n",
      "| serial_timesteps   | 81920        |\n",
      "| time_elapsed       | 3.76e+03     |\n",
      "| total_timesteps    | 81920        |\n",
      "| value_loss         | 0.95423746   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.021028407 |\n",
      "| clipfrac           | 0.17382812  |\n",
      "| explained_variance | 0.987       |\n",
      "| fps                | 18          |\n",
      "| n_updates          | 650         |\n",
      "| policy_entropy     | 0.4434812   |\n",
      "| policy_loss        | 0.006320627 |\n",
      "| serial_timesteps   | 83200       |\n",
      "| time_elapsed       | 3.83e+03    |\n",
      "| total_timesteps    | 83200       |\n",
      "| value_loss         | 7.889606    |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.012312297   |\n",
      "| clipfrac           | 0.1328125     |\n",
      "| explained_variance | 0.368         |\n",
      "| fps                | 20            |\n",
      "| n_updates          | 660           |\n",
      "| policy_entropy     | 0.6984813     |\n",
      "| policy_loss        | -0.0029256954 |\n",
      "| serial_timesteps   | 84480         |\n",
      "| time_elapsed       | 3.9e+03       |\n",
      "| total_timesteps    | 84480         |\n",
      "| value_loss         | 3.156055      |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0025864528  |\n",
      "| clipfrac           | 0.0390625     |\n",
      "| explained_variance | 0.997         |\n",
      "| fps                | 15            |\n",
      "| n_updates          | 670           |\n",
      "| policy_entropy     | 0.664211      |\n",
      "| policy_loss        | -0.0042667277 |\n",
      "| serial_timesteps   | 85760         |\n",
      "| time_elapsed       | 3.97e+03      |\n",
      "| total_timesteps    | 85760         |\n",
      "| value_loss         | 1.7837579     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.04573926   |\n",
      "| clipfrac           | 0.3046875    |\n",
      "| explained_variance | 0.738        |\n",
      "| fps                | 21           |\n",
      "| n_updates          | 680          |\n",
      "| policy_entropy     | 0.682268     |\n",
      "| policy_loss        | 0.0066902526 |\n",
      "| serial_timesteps   | 87040        |\n",
      "| time_elapsed       | 4.04e+03     |\n",
      "| total_timesteps    | 87040        |\n",
      "| value_loss         | 0.867653     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.012942871  |\n",
      "| clipfrac           | 0.17773438   |\n",
      "| explained_variance | 0.992        |\n",
      "| fps                | 15           |\n",
      "| n_updates          | 690          |\n",
      "| policy_entropy     | 0.70022047   |\n",
      "| policy_loss        | -0.012625041 |\n",
      "| serial_timesteps   | 88320        |\n",
      "| time_elapsed       | 4.11e+03     |\n",
      "| total_timesteps    | 88320        |\n",
      "| value_loss         | 0.8552735    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.011857941   |\n",
      "| clipfrac           | 0.1640625     |\n",
      "| explained_variance | 0.997         |\n",
      "| fps                | 20            |\n",
      "| n_updates          | 700           |\n",
      "| policy_entropy     | 0.45128244    |\n",
      "| policy_loss        | -0.0015573204 |\n",
      "| serial_timesteps   | 89600         |\n",
      "| time_elapsed       | 4.18e+03      |\n",
      "| total_timesteps    | 89600         |\n",
      "| value_loss         | 1.1033762     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.021539291 |\n",
      "| clipfrac           | 0.203125    |\n",
      "| explained_variance | 0.995       |\n",
      "| fps                | 19          |\n",
      "| n_updates          | 710         |\n",
      "| policy_entropy     | 0.5413573   |\n",
      "| policy_loss        | 0.016622795 |\n",
      "| serial_timesteps   | 90880       |\n",
      "| time_elapsed       | 4.25e+03    |\n",
      "| total_timesteps    | 90880       |\n",
      "| value_loss         | 1.6026477   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.016553797   |\n",
      "| clipfrac           | 0.12890625    |\n",
      "| explained_variance | 0.998         |\n",
      "| fps                | 16            |\n",
      "| n_updates          | 720           |\n",
      "| policy_entropy     | 0.78978795    |\n",
      "| policy_loss        | -0.0030433238 |\n",
      "| serial_timesteps   | 92160         |\n",
      "| time_elapsed       | 4.32e+03      |\n",
      "| total_timesteps    | 92160         |\n",
      "| value_loss         | 1.0410258     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.020962872  |\n",
      "| clipfrac           | 0.203125     |\n",
      "| explained_variance | 0.634        |\n",
      "| fps                | 20           |\n",
      "| n_updates          | 730          |\n",
      "| policy_entropy     | 0.43113807   |\n",
      "| policy_loss        | -0.006186213 |\n",
      "| serial_timesteps   | 93440        |\n",
      "| time_elapsed       | 4.39e+03     |\n",
      "| total_timesteps    | 93440        |\n",
      "| value_loss         | 4.7393193    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0037801038  |\n",
      "| clipfrac           | 0.04296875    |\n",
      "| explained_variance | 0.994         |\n",
      "| fps                | 15            |\n",
      "| n_updates          | 740           |\n",
      "| policy_entropy     | 0.85345477    |\n",
      "| policy_loss        | -0.0067483867 |\n",
      "| serial_timesteps   | 94720         |\n",
      "| time_elapsed       | 4.46e+03      |\n",
      "| total_timesteps    | 94720         |\n",
      "| value_loss         | 1.3075337     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0050811116 |\n",
      "| clipfrac           | 0.064453125  |\n",
      "| explained_variance | 0.867        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 750          |\n",
      "| policy_entropy     | 0.7905215    |\n",
      "| policy_loss        | -0.001124114 |\n",
      "| serial_timesteps   | 96000        |\n",
      "| time_elapsed       | 4.51e+03     |\n",
      "| total_timesteps    | 96000        |\n",
      "| value_loss         | 0.9537385    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.044665918  |\n",
      "| clipfrac           | 0.2578125    |\n",
      "| explained_variance | 0.99         |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 760          |\n",
      "| policy_entropy     | 1.1351924    |\n",
      "| policy_loss        | -0.026106428 |\n",
      "| serial_timesteps   | 97280        |\n",
      "| time_elapsed       | 4.56e+03     |\n",
      "| total_timesteps    | 97280        |\n",
      "| value_loss         | 7.3246045    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.020044694   |\n",
      "| clipfrac           | 0.29882812    |\n",
      "| explained_variance | 0.871         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 770           |\n",
      "| policy_entropy     | 0.88712615    |\n",
      "| policy_loss        | -0.0030863069 |\n",
      "| serial_timesteps   | 98560         |\n",
      "| time_elapsed       | 4.61e+03      |\n",
      "| total_timesteps    | 98560         |\n",
      "| value_loss         | 1.9584289     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.029658      |\n",
      "| clipfrac           | 0.31054688    |\n",
      "| explained_variance | 0.893         |\n",
      "| fps                | 26            |\n",
      "| n_updates          | 780           |\n",
      "| policy_entropy     | 0.57545865    |\n",
      "| policy_loss        | -0.0068461103 |\n",
      "| serial_timesteps   | 99840         |\n",
      "| time_elapsed       | 4.66e+03      |\n",
      "| total_timesteps    | 99840         |\n",
      "| value_loss         | 2.1262453     |\n",
      "--------------------------------------\n",
      "Eval num_timesteps=100000, episode_reward=453.90 +/- 8.87\n",
      "Episode length: 587.20 +/- 1.60\n",
      "------------------------------------\n",
      "| approxkl           | 0.058071643 |\n",
      "| clipfrac           | 0.35351562  |\n",
      "| explained_variance | 0.995       |\n",
      "| fps                | 23          |\n",
      "| n_updates          | 790         |\n",
      "| policy_entropy     | 0.8123756   |\n",
      "| policy_loss        | 0.031638607 |\n",
      "| serial_timesteps   | 101120      |\n",
      "| time_elapsed       | 4.81e+03    |\n",
      "| total_timesteps    | 101120      |\n",
      "| value_loss         | 1.397758    |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.005262208  |\n",
      "| clipfrac           | 0.060546875  |\n",
      "| explained_variance | 0.941        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 800          |\n",
      "| policy_entropy     | 0.51379424   |\n",
      "| policy_loss        | -0.012508126 |\n",
      "| serial_timesteps   | 102400       |\n",
      "| time_elapsed       | 4.85e+03     |\n",
      "| total_timesteps    | 102400       |\n",
      "| value_loss         | 1.0288998    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0049879514  |\n",
      "| clipfrac           | 0.0625        |\n",
      "| explained_variance | 0.706         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 810           |\n",
      "| policy_entropy     | 0.3821276     |\n",
      "| policy_loss        | -0.0019534826 |\n",
      "| serial_timesteps   | 103680        |\n",
      "| time_elapsed       | 4.9e+03       |\n",
      "| total_timesteps    | 103680        |\n",
      "| value_loss         | 4.764271      |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.010713105  |\n",
      "| clipfrac           | 0.13085938   |\n",
      "| explained_variance | 0.995        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 820          |\n",
      "| policy_entropy     | 0.8136645    |\n",
      "| policy_loss        | -0.004622898 |\n",
      "| serial_timesteps   | 104960       |\n",
      "| time_elapsed       | 4.95e+03     |\n",
      "| total_timesteps    | 104960       |\n",
      "| value_loss         | 1.1905503    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.008863953 |\n",
      "| clipfrac           | 0.14648438  |\n",
      "| explained_variance | 0.931       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 830         |\n",
      "| policy_entropy     | 0.5230821   |\n",
      "| policy_loss        | 0.002577581 |\n",
      "| serial_timesteps   | 106240      |\n",
      "| time_elapsed       | 5e+03       |\n",
      "| total_timesteps    | 106240      |\n",
      "| value_loss         | 2.4974484   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0039083036 |\n",
      "| clipfrac           | 0.048828125  |\n",
      "| explained_variance | 0.905        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 840          |\n",
      "| policy_entropy     | 0.3569783    |\n",
      "| policy_loss        | 0.004155222  |\n",
      "| serial_timesteps   | 107520       |\n",
      "| time_elapsed       | 5.04e+03     |\n",
      "| total_timesteps    | 107520       |\n",
      "| value_loss         | 0.86314774   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.025532916  |\n",
      "| clipfrac           | 0.1328125    |\n",
      "| explained_variance | 0.992        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 850          |\n",
      "| policy_entropy     | 0.6352306    |\n",
      "| policy_loss        | 0.0005540814 |\n",
      "| serial_timesteps   | 108800       |\n",
      "| time_elapsed       | 5.09e+03     |\n",
      "| total_timesteps    | 108800       |\n",
      "| value_loss         | 1.0303959    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.09427533  |\n",
      "| clipfrac           | 0.30664062  |\n",
      "| explained_variance | 0.997       |\n",
      "| fps                | 28          |\n",
      "| n_updates          | 860         |\n",
      "| policy_entropy     | 0.7359796   |\n",
      "| policy_loss        | -0.02635399 |\n",
      "| serial_timesteps   | 110080      |\n",
      "| time_elapsed       | 5.14e+03    |\n",
      "| total_timesteps    | 110080      |\n",
      "| value_loss         | 1.9801002   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.03994524    |\n",
      "| clipfrac           | 0.2890625     |\n",
      "| explained_variance | 0.898         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 870           |\n",
      "| policy_entropy     | 0.7631938     |\n",
      "| policy_loss        | -0.0005120889 |\n",
      "| serial_timesteps   | 111360        |\n",
      "| time_elapsed       | 5.18e+03      |\n",
      "| total_timesteps    | 111360        |\n",
      "| value_loss         | 0.42006192    |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.036125015 |\n",
      "| clipfrac           | 0.27929688  |\n",
      "| explained_variance | 0.995       |\n",
      "| fps                | 25          |\n",
      "| n_updates          | 880         |\n",
      "| policy_entropy     | 0.50866395  |\n",
      "| policy_loss        | 0.01357261  |\n",
      "| serial_timesteps   | 112640      |\n",
      "| time_elapsed       | 5.23e+03    |\n",
      "| total_timesteps    | 112640      |\n",
      "| value_loss         | 0.55015624  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.03015504   |\n",
      "| clipfrac           | 0.18945312   |\n",
      "| explained_variance | 0.995        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 890          |\n",
      "| policy_entropy     | 0.3929823    |\n",
      "| policy_loss        | 0.0136762895 |\n",
      "| serial_timesteps   | 113920       |\n",
      "| time_elapsed       | 5.28e+03     |\n",
      "| total_timesteps    | 113920       |\n",
      "| value_loss         | 1.9676051    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.061056416 |\n",
      "| clipfrac           | 0.5410156   |\n",
      "| explained_variance | 0.0908      |\n",
      "| fps                | 31          |\n",
      "| n_updates          | 900         |\n",
      "| policy_entropy     | 0.66719884  |\n",
      "| policy_loss        | 0.056020558 |\n",
      "| serial_timesteps   | 115200      |\n",
      "| time_elapsed       | 5.33e+03    |\n",
      "| total_timesteps    | 115200      |\n",
      "| value_loss         | 0.4536926   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.01171946   |\n",
      "| clipfrac           | 0.12890625   |\n",
      "| explained_variance | 0.987        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 910          |\n",
      "| policy_entropy     | 0.54818845   |\n",
      "| policy_loss        | -0.004410883 |\n",
      "| serial_timesteps   | 116480       |\n",
      "| time_elapsed       | 5.37e+03     |\n",
      "| total_timesteps    | 116480       |\n",
      "| value_loss         | 0.4609935    |\n",
      "-------------------------------------\n",
      "-----------------------------------\n",
      "| approxkl           | 0.03848404 |\n",
      "| clipfrac           | 0.18164062 |\n",
      "| explained_variance | 0.918      |\n",
      "| fps                | 21         |\n",
      "| n_updates          | 920        |\n",
      "| policy_entropy     | 0.3418532  |\n",
      "| policy_loss        | 0.03327635 |\n",
      "| serial_timesteps   | 117760     |\n",
      "| time_elapsed       | 5.42e+03   |\n",
      "| total_timesteps    | 117760     |\n",
      "| value_loss         | 7.1379724  |\n",
      "-----------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.031683166  |\n",
      "| clipfrac           | 0.29296875   |\n",
      "| explained_variance | 0.74         |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 930          |\n",
      "| policy_entropy     | 0.70754665   |\n",
      "| policy_loss        | -0.005292474 |\n",
      "| serial_timesteps   | 119040       |\n",
      "| time_elapsed       | 5.47e+03     |\n",
      "| total_timesteps    | 119040       |\n",
      "| value_loss         | 4.8794246    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.02064833   |\n",
      "| clipfrac           | 0.2265625    |\n",
      "| explained_variance | -0.529       |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 940          |\n",
      "| policy_entropy     | 0.4661889    |\n",
      "| policy_loss        | 0.0056781364 |\n",
      "| serial_timesteps   | 120320       |\n",
      "| time_elapsed       | 5.51e+03     |\n",
      "| total_timesteps    | 120320       |\n",
      "| value_loss         | 12.554866    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.004248987  |\n",
      "| clipfrac           | 0.052734375  |\n",
      "| explained_variance | 0.984        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 950          |\n",
      "| policy_entropy     | 0.34756774   |\n",
      "| policy_loss        | -0.006581042 |\n",
      "| serial_timesteps   | 121600       |\n",
      "| time_elapsed       | 5.56e+03     |\n",
      "| total_timesteps    | 121600       |\n",
      "| value_loss         | 1.2819971    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.012625635   |\n",
      "| clipfrac           | 0.14257812    |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 960           |\n",
      "| policy_entropy     | 0.30716124    |\n",
      "| policy_loss        | 0.00037626398 |\n",
      "| serial_timesteps   | 122880        |\n",
      "| time_elapsed       | 5.61e+03      |\n",
      "| total_timesteps    | 122880        |\n",
      "| value_loss         | 1.0688903     |\n",
      "--------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------\n",
      "| approxkl           | 0.0075803366  |\n",
      "| clipfrac           | 0.095703125   |\n",
      "| explained_variance | 0.762         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 970           |\n",
      "| policy_entropy     | 0.4460855     |\n",
      "| policy_loss        | -0.0015815371 |\n",
      "| serial_timesteps   | 124160        |\n",
      "| time_elapsed       | 5.65e+03      |\n",
      "| total_timesteps    | 124160        |\n",
      "| value_loss         | 0.3966958     |\n",
      "--------------------------------------\n",
      "Eval num_timesteps=125000, episode_reward=552.10 +/- 1.07\n",
      "Episode length: 592.60 +/- 0.49\n",
      "New best mean reward!\n",
      "-------------------------------------\n",
      "| approxkl           | 0.026498005  |\n",
      "| clipfrac           | 0.17578125   |\n",
      "| explained_variance | 0.994        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 980          |\n",
      "| policy_entropy     | 0.43416166   |\n",
      "| policy_loss        | 0.0037130606 |\n",
      "| serial_timesteps   | 125440       |\n",
      "| time_elapsed       | 5.8e+03      |\n",
      "| total_timesteps    | 125440       |\n",
      "| value_loss         | 0.627022     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0109977815 |\n",
      "| clipfrac           | 0.09765625   |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 990          |\n",
      "| policy_entropy     | 0.30606782   |\n",
      "| policy_loss        | 0.016567647  |\n",
      "| serial_timesteps   | 126720       |\n",
      "| time_elapsed       | 5.85e+03     |\n",
      "| total_timesteps    | 126720       |\n",
      "| value_loss         | 0.84813905   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.006111089   |\n",
      "| clipfrac           | 0.0703125     |\n",
      "| explained_variance | 0.762         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 1000          |\n",
      "| policy_entropy     | 0.6260691     |\n",
      "| policy_loss        | -0.0026229543 |\n",
      "| serial_timesteps   | 128000        |\n",
      "| time_elapsed       | 5.9e+03       |\n",
      "| total_timesteps    | 128000        |\n",
      "| value_loss         | 0.1997304     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.021182558  |\n",
      "| clipfrac           | 0.16601562   |\n",
      "| explained_variance | 0.99         |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 1010         |\n",
      "| policy_entropy     | 0.39997822   |\n",
      "| policy_loss        | 0.0015700243 |\n",
      "| serial_timesteps   | 129280       |\n",
      "| time_elapsed       | 5.95e+03     |\n",
      "| total_timesteps    | 129280       |\n",
      "| value_loss         | 0.36462277   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.005234716   |\n",
      "| clipfrac           | 0.083984375   |\n",
      "| explained_variance | 0.991         |\n",
      "| fps                | 24            |\n",
      "| n_updates          | 1020          |\n",
      "| policy_entropy     | 0.3963146     |\n",
      "| policy_loss        | -0.0051883264 |\n",
      "| serial_timesteps   | 130560        |\n",
      "| time_elapsed       | 5.99e+03      |\n",
      "| total_timesteps    | 130560        |\n",
      "| value_loss         | 3.0489688     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.008007782 |\n",
      "| clipfrac           | 0.111328125 |\n",
      "| explained_variance | 0.0105      |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1030        |\n",
      "| policy_entropy     | 0.57415265  |\n",
      "| policy_loss        | 0.002545591 |\n",
      "| serial_timesteps   | 131840      |\n",
      "| time_elapsed       | 6.04e+03    |\n",
      "| total_timesteps    | 131840      |\n",
      "| value_loss         | 0.77790725  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.008083329  |\n",
      "| clipfrac           | 0.12890625   |\n",
      "| explained_variance | 0.941        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 1040         |\n",
      "| policy_entropy     | 0.5474663    |\n",
      "| policy_loss        | -0.009463366 |\n",
      "| serial_timesteps   | 133120       |\n",
      "| time_elapsed       | 6.09e+03     |\n",
      "| total_timesteps    | 133120       |\n",
      "| value_loss         | 0.42504665   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.006156656  |\n",
      "| clipfrac           | 0.0859375    |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 1050         |\n",
      "| policy_entropy     | 0.41263485   |\n",
      "| policy_loss        | 0.0009454745 |\n",
      "| serial_timesteps   | 134400       |\n",
      "| time_elapsed       | 6.13e+03     |\n",
      "| total_timesteps    | 134400       |\n",
      "| value_loss         | 0.20104074   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.008288822 |\n",
      "| clipfrac           | 0.115234375 |\n",
      "| explained_variance | 0.814       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1060        |\n",
      "| policy_entropy     | 0.6501071   |\n",
      "| policy_loss        | 0.008314246 |\n",
      "| serial_timesteps   | 135680      |\n",
      "| time_elapsed       | 6.18e+03    |\n",
      "| total_timesteps    | 135680      |\n",
      "| value_loss         | 3.524724    |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.015670978   |\n",
      "| clipfrac           | 0.19921875    |\n",
      "| explained_variance | 0.953         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 1070          |\n",
      "| policy_entropy     | 0.45598143    |\n",
      "| policy_loss        | 0.00087975466 |\n",
      "| serial_timesteps   | 136960        |\n",
      "| time_elapsed       | 6.23e+03      |\n",
      "| total_timesteps    | 136960        |\n",
      "| value_loss         | 0.3830007     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.01087034   |\n",
      "| clipfrac           | 0.14453125   |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 1080         |\n",
      "| policy_entropy     | 0.42237467   |\n",
      "| policy_loss        | 0.0049131406 |\n",
      "| serial_timesteps   | 138240       |\n",
      "| time_elapsed       | 6.28e+03     |\n",
      "| total_timesteps    | 138240       |\n",
      "| value_loss         | 0.28197473   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.026933882 |\n",
      "| clipfrac           | 0.35351562  |\n",
      "| explained_variance | 0.985       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1090        |\n",
      "| policy_entropy     | 0.5571667   |\n",
      "| policy_loss        | 0.019019697 |\n",
      "| serial_timesteps   | 139520      |\n",
      "| time_elapsed       | 6.32e+03    |\n",
      "| total_timesteps    | 139520      |\n",
      "| value_loss         | 1.7141871   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.011611633 |\n",
      "| clipfrac           | 0.15039062  |\n",
      "| explained_variance | 0.975       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1100        |\n",
      "| policy_entropy     | 0.45684364  |\n",
      "| policy_loss        | 0.002228504 |\n",
      "| serial_timesteps   | 140800      |\n",
      "| time_elapsed       | 6.37e+03    |\n",
      "| total_timesteps    | 140800      |\n",
      "| value_loss         | 0.70769835  |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.018648526   |\n",
      "| clipfrac           | 0.23242188    |\n",
      "| explained_variance | 0.996         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 1110          |\n",
      "| policy_entropy     | 0.51075       |\n",
      "| policy_loss        | -0.0040318887 |\n",
      "| serial_timesteps   | 142080        |\n",
      "| time_elapsed       | 6.42e+03      |\n",
      "| total_timesteps    | 142080        |\n",
      "| value_loss         | 0.58512986    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.01954922    |\n",
      "| clipfrac           | 0.095703125   |\n",
      "| explained_variance | 0.998         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 1120          |\n",
      "| policy_entropy     | 0.19286853    |\n",
      "| policy_loss        | -0.0011268887 |\n",
      "| serial_timesteps   | 143360        |\n",
      "| time_elapsed       | 6.46e+03      |\n",
      "| total_timesteps    | 143360        |\n",
      "| value_loss         | 1.6304264     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.016226871 |\n",
      "| clipfrac           | 0.19921875  |\n",
      "| explained_variance | 0.784       |\n",
      "| fps                | 31          |\n",
      "| n_updates          | 1130        |\n",
      "| policy_entropy     | 0.37586096  |\n",
      "| policy_loss        | 0.010880625 |\n",
      "| serial_timesteps   | 144640      |\n",
      "| time_elapsed       | 6.51e+03    |\n",
      "| total_timesteps    | 144640      |\n",
      "| value_loss         | 2.0507278   |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------\n",
      "| approxkl           | 0.017482249 |\n",
      "| clipfrac           | 0.24023438  |\n",
      "| explained_variance | 0.968       |\n",
      "| fps                | 25          |\n",
      "| n_updates          | 1140        |\n",
      "| policy_entropy     | 0.42198876  |\n",
      "| policy_loss        | 0.008142809 |\n",
      "| serial_timesteps   | 145920      |\n",
      "| time_elapsed       | 6.56e+03    |\n",
      "| total_timesteps    | 145920      |\n",
      "| value_loss         | 1.4827374   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0031602485  |\n",
      "| clipfrac           | 0.041015625   |\n",
      "| explained_variance | 0.99          |\n",
      "| fps                | 25            |\n",
      "| n_updates          | 1150          |\n",
      "| policy_entropy     | 0.17006433    |\n",
      "| policy_loss        | -0.0037131023 |\n",
      "| serial_timesteps   | 147200        |\n",
      "| time_elapsed       | 6.61e+03      |\n",
      "| total_timesteps    | 147200        |\n",
      "| value_loss         | 4.8871517     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.025604919 |\n",
      "| clipfrac           | 0.34179688  |\n",
      "| explained_variance | 0.947       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1160        |\n",
      "| policy_entropy     | 0.55417943  |\n",
      "| policy_loss        | 0.010744437 |\n",
      "| serial_timesteps   | 148480      |\n",
      "| time_elapsed       | 6.65e+03    |\n",
      "| total_timesteps    | 148480      |\n",
      "| value_loss         | 0.13565949  |\n",
      "------------------------------------\n",
      "-----------------------------------\n",
      "| approxkl           | 0.03244827 |\n",
      "| clipfrac           | 0.23632812 |\n",
      "| explained_variance | 0.977      |\n",
      "| fps                | 26         |\n",
      "| n_updates          | 1170       |\n",
      "| policy_entropy     | 0.5313211  |\n",
      "| policy_loss        | 0.02252198 |\n",
      "| serial_timesteps   | 149760     |\n",
      "| time_elapsed       | 6.7e+03    |\n",
      "| total_timesteps    | 149760     |\n",
      "| value_loss         | 0.43929565 |\n",
      "-----------------------------------\n",
      "Eval num_timesteps=150000, episode_reward=547.10 +/- 1.93\n",
      "Episode length: 589.00 +/- 1.90\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0012972802  |\n",
      "| clipfrac           | 0.013671875   |\n",
      "| explained_variance | 0.996         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 1180          |\n",
      "| policy_entropy     | 0.3889793     |\n",
      "| policy_loss        | -0.0027521458 |\n",
      "| serial_timesteps   | 151040        |\n",
      "| time_elapsed       | 6.85e+03      |\n",
      "| total_timesteps    | 151040        |\n",
      "| value_loss         | 0.54366666    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.014882618   |\n",
      "| clipfrac           | 0.21679688    |\n",
      "| explained_variance | 0.995         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 1190          |\n",
      "| policy_entropy     | 0.6255093     |\n",
      "| policy_loss        | -0.0012823269 |\n",
      "| serial_timesteps   | 152320        |\n",
      "| time_elapsed       | 6.9e+03       |\n",
      "| total_timesteps    | 152320        |\n",
      "| value_loss         | 0.95022696    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.029758153   |\n",
      "| clipfrac           | 0.13476562    |\n",
      "| explained_variance | 0.951         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 1200          |\n",
      "| policy_entropy     | 0.40241432    |\n",
      "| policy_loss        | -0.0062571233 |\n",
      "| serial_timesteps   | 153600        |\n",
      "| time_elapsed       | 6.94e+03      |\n",
      "| total_timesteps    | 153600        |\n",
      "| value_loss         | 0.33946735    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.003956248   |\n",
      "| clipfrac           | 0.0546875     |\n",
      "| explained_variance | 0.994         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 1210          |\n",
      "| policy_entropy     | 0.4412415     |\n",
      "| policy_loss        | -0.0029587299 |\n",
      "| serial_timesteps   | 154880        |\n",
      "| time_elapsed       | 6.99e+03      |\n",
      "| total_timesteps    | 154880        |\n",
      "| value_loss         | 0.5306521     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.031057838  |\n",
      "| clipfrac           | 0.26367188   |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 1220         |\n",
      "| policy_entropy     | 0.4675238    |\n",
      "| policy_loss        | -0.008970363 |\n",
      "| serial_timesteps   | 156160       |\n",
      "| time_elapsed       | 7.04e+03     |\n",
      "| total_timesteps    | 156160       |\n",
      "| value_loss         | 0.8467124    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.088313565 |\n",
      "| clipfrac           | 0.5332031   |\n",
      "| explained_variance | 0.951       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1230        |\n",
      "| policy_entropy     | 0.4922265   |\n",
      "| policy_loss        | 0.032348134 |\n",
      "| serial_timesteps   | 157440      |\n",
      "| time_elapsed       | 7.08e+03    |\n",
      "| total_timesteps    | 157440      |\n",
      "| value_loss         | 0.7822207   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.03851949  |\n",
      "| clipfrac           | 0.4140625   |\n",
      "| explained_variance | 0.99        |\n",
      "| fps                | 25          |\n",
      "| n_updates          | 1240        |\n",
      "| policy_entropy     | 0.7824724   |\n",
      "| policy_loss        | 0.031487424 |\n",
      "| serial_timesteps   | 158720      |\n",
      "| time_elapsed       | 7.13e+03    |\n",
      "| total_timesteps    | 158720      |\n",
      "| value_loss         | 0.34610796  |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.004869942   |\n",
      "| clipfrac           | 0.078125      |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 27            |\n",
      "| n_updates          | 1250          |\n",
      "| policy_entropy     | 0.6228608     |\n",
      "| policy_loss        | -0.0058626356 |\n",
      "| serial_timesteps   | 160000        |\n",
      "| time_elapsed       | 7.18e+03      |\n",
      "| total_timesteps    | 160000        |\n",
      "| value_loss         | 0.29501268    |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.01713156  |\n",
      "| clipfrac           | 0.24023438  |\n",
      "| explained_variance | 0.946       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1260        |\n",
      "| policy_entropy     | 0.72501844  |\n",
      "| policy_loss        | 0.007162847 |\n",
      "| serial_timesteps   | 161280      |\n",
      "| time_elapsed       | 7.23e+03    |\n",
      "| total_timesteps    | 161280      |\n",
      "| value_loss         | 1.0640061   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.041689765  |\n",
      "| clipfrac           | 0.29492188   |\n",
      "| explained_variance | 0.907        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 1270         |\n",
      "| policy_entropy     | 0.553441     |\n",
      "| policy_loss        | -0.006159697 |\n",
      "| serial_timesteps   | 162560       |\n",
      "| time_elapsed       | 7.27e+03     |\n",
      "| total_timesteps    | 162560       |\n",
      "| value_loss         | 5.958192     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0022173857 |\n",
      "| clipfrac           | 0.029296875  |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 1280         |\n",
      "| policy_entropy     | 0.40875158   |\n",
      "| policy_loss        | -0.003908269 |\n",
      "| serial_timesteps   | 163840       |\n",
      "| time_elapsed       | 7.32e+03     |\n",
      "| total_timesteps    | 163840       |\n",
      "| value_loss         | 0.6304116    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.011288213  |\n",
      "| clipfrac           | 0.14453125   |\n",
      "| explained_variance | 0.889        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 1290         |\n",
      "| policy_entropy     | 0.47492993   |\n",
      "| policy_loss        | -0.004910833 |\n",
      "| serial_timesteps   | 165120       |\n",
      "| time_elapsed       | 7.37e+03     |\n",
      "| total_timesteps    | 165120       |\n",
      "| value_loss         | 1.1313457    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0189378    |\n",
      "| clipfrac           | 0.17773438   |\n",
      "| explained_variance | 0.961        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 1300         |\n",
      "| policy_entropy     | 0.78814584   |\n",
      "| policy_loss        | -0.028380351 |\n",
      "| serial_timesteps   | 166400       |\n",
      "| time_elapsed       | 7.41e+03     |\n",
      "| total_timesteps    | 166400       |\n",
      "| value_loss         | 4.4026737    |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------\n",
      "| approxkl           | 0.0057577365  |\n",
      "| clipfrac           | 0.087890625   |\n",
      "| explained_variance | 0.889         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 1310          |\n",
      "| policy_entropy     | 0.731407      |\n",
      "| policy_loss        | -0.0060996762 |\n",
      "| serial_timesteps   | 167680        |\n",
      "| time_elapsed       | 7.46e+03      |\n",
      "| total_timesteps    | 167680        |\n",
      "| value_loss         | 0.9317972     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.034432627   |\n",
      "| clipfrac           | 0.41210938    |\n",
      "| explained_variance | 0.576         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 1320          |\n",
      "| policy_entropy     | 0.97747695    |\n",
      "| policy_loss        | -0.0016433739 |\n",
      "| serial_timesteps   | 168960        |\n",
      "| time_elapsed       | 7.51e+03      |\n",
      "| total_timesteps    | 168960        |\n",
      "| value_loss         | 179.83514     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.02227626    |\n",
      "| clipfrac           | 0.27734375    |\n",
      "| explained_variance | 0.42          |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 1330          |\n",
      "| policy_entropy     | 1.1082305     |\n",
      "| policy_loss        | -0.0010911473 |\n",
      "| serial_timesteps   | 170240        |\n",
      "| time_elapsed       | 7.55e+03      |\n",
      "| total_timesteps    | 170240        |\n",
      "| value_loss         | 1.7432629     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.014176819   |\n",
      "| clipfrac           | 0.21875       |\n",
      "| explained_variance | 0.987         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 1340          |\n",
      "| policy_entropy     | 1.0201447     |\n",
      "| policy_loss        | -0.0045372327 |\n",
      "| serial_timesteps   | 171520        |\n",
      "| time_elapsed       | 7.6e+03       |\n",
      "| total_timesteps    | 171520        |\n",
      "| value_loss         | 3.1318824     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.03211426  |\n",
      "| clipfrac           | 0.47070312  |\n",
      "| explained_variance | 0.915       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1350        |\n",
      "| policy_entropy     | 0.97633576  |\n",
      "| policy_loss        | 0.019265302 |\n",
      "| serial_timesteps   | 172800      |\n",
      "| time_elapsed       | 7.65e+03    |\n",
      "| total_timesteps    | 172800      |\n",
      "| value_loss         | 0.5517187   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0044813007  |\n",
      "| clipfrac           | 0.0546875     |\n",
      "| explained_variance | 0.915         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 1360          |\n",
      "| policy_entropy     | 0.66829205    |\n",
      "| policy_loss        | -0.0010201875 |\n",
      "| serial_timesteps   | 174080        |\n",
      "| time_elapsed       | 7.69e+03      |\n",
      "| total_timesteps    | 174080        |\n",
      "| value_loss         | 1.0082839     |\n",
      "--------------------------------------\n",
      "Eval num_timesteps=175000, episode_reward=542.30 +/- 2.66\n",
      "Episode length: 587.00 +/- 0.89\n",
      "-------------------------------------\n",
      "| approxkl           | 0.031303734  |\n",
      "| clipfrac           | 0.18945312   |\n",
      "| explained_variance | 0.981        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 1370         |\n",
      "| policy_entropy     | 0.6029394    |\n",
      "| policy_loss        | -0.007931516 |\n",
      "| serial_timesteps   | 175360       |\n",
      "| time_elapsed       | 7.84e+03     |\n",
      "| total_timesteps    | 175360       |\n",
      "| value_loss         | 0.49876106   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.024773324  |\n",
      "| clipfrac           | 0.123046875  |\n",
      "| explained_variance | 0.993        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 1380         |\n",
      "| policy_entropy     | 0.66725796   |\n",
      "| policy_loss        | -0.024585383 |\n",
      "| serial_timesteps   | 176640       |\n",
      "| time_elapsed       | 7.89e+03     |\n",
      "| total_timesteps    | 176640       |\n",
      "| value_loss         | 1.2512525    |\n",
      "-------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.014490696    |\n",
      "| clipfrac           | 0.16015625     |\n",
      "| explained_variance | 0.946          |\n",
      "| fps                | 30             |\n",
      "| n_updates          | 1390           |\n",
      "| policy_entropy     | 0.5522666      |\n",
      "| policy_loss        | -0.00033865822 |\n",
      "| serial_timesteps   | 177920         |\n",
      "| time_elapsed       | 7.94e+03       |\n",
      "| total_timesteps    | 177920         |\n",
      "| value_loss         | 1.4377067      |\n",
      "---------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.019946203  |\n",
      "| clipfrac           | 0.25195312   |\n",
      "| explained_variance | 0.981        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 1400         |\n",
      "| policy_entropy     | 0.5343286    |\n",
      "| policy_loss        | -0.018305747 |\n",
      "| serial_timesteps   | 179200       |\n",
      "| time_elapsed       | 7.98e+03     |\n",
      "| total_timesteps    | 179200       |\n",
      "| value_loss         | 0.14875394   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.018918926 |\n",
      "| clipfrac           | 0.27929688  |\n",
      "| explained_variance | 0.835       |\n",
      "| fps                | 25          |\n",
      "| n_updates          | 1410        |\n",
      "| policy_entropy     | 0.66343796  |\n",
      "| policy_loss        | 0.0151864   |\n",
      "| serial_timesteps   | 180480      |\n",
      "| time_elapsed       | 8.03e+03    |\n",
      "| total_timesteps    | 180480      |\n",
      "| value_loss         | 84.98202    |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.026884895 |\n",
      "| clipfrac           | 0.14648438  |\n",
      "| explained_variance | 0.638       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1420        |\n",
      "| policy_entropy     | 0.37485865  |\n",
      "| policy_loss        | -0.01763953 |\n",
      "| serial_timesteps   | 181760      |\n",
      "| time_elapsed       | 8.08e+03    |\n",
      "| total_timesteps    | 181760      |\n",
      "| value_loss         | 64.318756   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.010589459 |\n",
      "| clipfrac           | 0.17773438  |\n",
      "| explained_variance | 0.98        |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1430        |\n",
      "| policy_entropy     | 0.7430089   |\n",
      "| policy_loss        | 0.003046211 |\n",
      "| serial_timesteps   | 183040      |\n",
      "| time_elapsed       | 8.12e+03    |\n",
      "| total_timesteps    | 183040      |\n",
      "| value_loss         | 1.8867649   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.061635762 |\n",
      "| clipfrac           | 0.40039062  |\n",
      "| explained_variance | 0.978       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1440        |\n",
      "| policy_entropy     | 0.6825582   |\n",
      "| policy_loss        | 0.01428628  |\n",
      "| serial_timesteps   | 184320      |\n",
      "| time_elapsed       | 8.17e+03    |\n",
      "| total_timesteps    | 184320      |\n",
      "| value_loss         | 0.22308066  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.013160542 |\n",
      "| clipfrac           | 0.125       |\n",
      "| explained_variance | 0.985       |\n",
      "| fps                | 22          |\n",
      "| n_updates          | 1450        |\n",
      "| policy_entropy     | 0.45107296  |\n",
      "| policy_loss        | 0.004933696 |\n",
      "| serial_timesteps   | 185600      |\n",
      "| time_elapsed       | 8.21e+03    |\n",
      "| total_timesteps    | 185600      |\n",
      "| value_loss         | 1.4244947   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.02202179  |\n",
      "| clipfrac           | 0.25976562  |\n",
      "| explained_variance | 0.993       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1460        |\n",
      "| policy_entropy     | 0.88365704  |\n",
      "| policy_loss        | 0.013918008 |\n",
      "| serial_timesteps   | 186880      |\n",
      "| time_elapsed       | 8.26e+03    |\n",
      "| total_timesteps    | 186880      |\n",
      "| value_loss         | 0.82934266  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.009742982  |\n",
      "| clipfrac           | 0.12695312   |\n",
      "| explained_variance | 0.217        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 1470         |\n",
      "| policy_entropy     | 0.67169493   |\n",
      "| policy_loss        | 0.0013200149 |\n",
      "| serial_timesteps   | 188160       |\n",
      "| time_elapsed       | 8.31e+03     |\n",
      "| total_timesteps    | 188160       |\n",
      "| value_loss         | 1.6925013    |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.0070767403 |\n",
      "| clipfrac           | 0.07421875   |\n",
      "| explained_variance | 0.992        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 1480         |\n",
      "| policy_entropy     | 0.41920987   |\n",
      "| policy_loss        | 0.0040275757 |\n",
      "| serial_timesteps   | 189440       |\n",
      "| time_elapsed       | 8.35e+03     |\n",
      "| total_timesteps    | 189440       |\n",
      "| value_loss         | 0.5185732    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.006880123   |\n",
      "| clipfrac           | 0.076171875   |\n",
      "| explained_variance | 0.998         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 1490          |\n",
      "| policy_entropy     | 0.8877719     |\n",
      "| policy_loss        | -0.0041551283 |\n",
      "| serial_timesteps   | 190720        |\n",
      "| time_elapsed       | 8.4e+03       |\n",
      "| total_timesteps    | 190720        |\n",
      "| value_loss         | 1.1686512     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0066086724  |\n",
      "| clipfrac           | 0.0703125     |\n",
      "| explained_variance | 0.755         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 1500          |\n",
      "| policy_entropy     | 0.53070563    |\n",
      "| policy_loss        | -0.0061969142 |\n",
      "| serial_timesteps   | 192000        |\n",
      "| time_elapsed       | 8.45e+03      |\n",
      "| total_timesteps    | 192000        |\n",
      "| value_loss         | 1.0482079     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.02750209   |\n",
      "| clipfrac           | 0.2421875    |\n",
      "| explained_variance | 0.978        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 1510         |\n",
      "| policy_entropy     | 0.708995     |\n",
      "| policy_loss        | -0.010848442 |\n",
      "| serial_timesteps   | 193280       |\n",
      "| time_elapsed       | 8.49e+03     |\n",
      "| total_timesteps    | 193280       |\n",
      "| value_loss         | 7.393588     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.009631129  |\n",
      "| clipfrac           | 0.12109375   |\n",
      "| explained_variance | 0.978        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 1520         |\n",
      "| policy_entropy     | 0.46399415   |\n",
      "| policy_loss        | 0.0038273518 |\n",
      "| serial_timesteps   | 194560       |\n",
      "| time_elapsed       | 8.54e+03     |\n",
      "| total_timesteps    | 194560       |\n",
      "| value_loss         | 0.42310172   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.030731555 |\n",
      "| clipfrac           | 0.31054688  |\n",
      "| explained_variance | 0.971       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 1530        |\n",
      "| policy_entropy     | 0.41097277  |\n",
      "| policy_loss        | 0.02554556  |\n",
      "| serial_timesteps   | 195840      |\n",
      "| time_elapsed       | 8.59e+03    |\n",
      "| total_timesteps    | 195840      |\n",
      "| value_loss         | 0.3023957   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.021489494   |\n",
      "| clipfrac           | 0.115234375   |\n",
      "| explained_variance | 0.992         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 1540          |\n",
      "| policy_entropy     | 0.35862958    |\n",
      "| policy_loss        | -0.0049280077 |\n",
      "| serial_timesteps   | 197120        |\n",
      "| time_elapsed       | 8.64e+03      |\n",
      "| total_timesteps    | 197120        |\n",
      "| value_loss         | 1.783073      |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.011244539  |\n",
      "| clipfrac           | 0.15039062   |\n",
      "| explained_variance | 0.989        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 1550         |\n",
      "| policy_entropy     | 0.5217037    |\n",
      "| policy_loss        | -0.006093622 |\n",
      "| serial_timesteps   | 198400       |\n",
      "| time_elapsed       | 8.68e+03     |\n",
      "| total_timesteps    | 198400       |\n",
      "| value_loss         | 1.1582162    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.01223009  |\n",
      "| clipfrac           | 0.17382812  |\n",
      "| explained_variance | 0.954       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1560        |\n",
      "| policy_entropy     | 0.5662748   |\n",
      "| policy_loss        | 0.004617462 |\n",
      "| serial_timesteps   | 199680      |\n",
      "| time_elapsed       | 8.73e+03    |\n",
      "| total_timesteps    | 199680      |\n",
      "| value_loss         | 0.7909552   |\n",
      "------------------------------------\n",
      "Eval num_timesteps=200000, episode_reward=549.40 +/- 0.86\n",
      "Episode length: 586.40 +/- 0.49\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0085498085 |\n",
      "| clipfrac           | 0.15039062   |\n",
      "| explained_variance | 0.992        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 1570         |\n",
      "| policy_entropy     | 0.64911234   |\n",
      "| policy_loss        | -0.010922488 |\n",
      "| serial_timesteps   | 200960       |\n",
      "| time_elapsed       | 8.88e+03     |\n",
      "| total_timesteps    | 200960       |\n",
      "| value_loss         | 0.60348004   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.012655014   |\n",
      "| clipfrac           | 0.115234375   |\n",
      "| explained_variance | 0.915         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 1580          |\n",
      "| policy_entropy     | 0.7749645     |\n",
      "| policy_loss        | -0.0038481718 |\n",
      "| serial_timesteps   | 202240        |\n",
      "| time_elapsed       | 8.93e+03      |\n",
      "| total_timesteps    | 202240        |\n",
      "| value_loss         | 38.681446     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.04280836    |\n",
      "| clipfrac           | 0.2578125     |\n",
      "| explained_variance | 0.725         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 1590          |\n",
      "| policy_entropy     | 0.5489508     |\n",
      "| policy_loss        | 0.00028999476 |\n",
      "| serial_timesteps   | 203520        |\n",
      "| time_elapsed       | 8.97e+03      |\n",
      "| total_timesteps    | 203520        |\n",
      "| value_loss         | 0.43937904    |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.014391818 |\n",
      "| clipfrac           | 0.087890625 |\n",
      "| explained_variance | 0.998       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 1600        |\n",
      "| policy_entropy     | 0.5680571   |\n",
      "| policy_loss        | 0.010765861 |\n",
      "| serial_timesteps   | 204800      |\n",
      "| time_elapsed       | 9.02e+03    |\n",
      "| total_timesteps    | 204800      |\n",
      "| value_loss         | 1.1780866   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.056953456 |\n",
      "| clipfrac           | 0.28710938  |\n",
      "| explained_variance | 0.426       |\n",
      "| fps                | 28          |\n",
      "| n_updates          | 1610        |\n",
      "| policy_entropy     | 0.705665    |\n",
      "| policy_loss        | 0.012844517 |\n",
      "| serial_timesteps   | 206080      |\n",
      "| time_elapsed       | 9.07e+03    |\n",
      "| total_timesteps    | 206080      |\n",
      "| value_loss         | 15.575781   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0106437635 |\n",
      "| clipfrac           | 0.103515625  |\n",
      "| explained_variance | 0.93         |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 1620         |\n",
      "| policy_entropy     | 0.6191636    |\n",
      "| policy_loss        | 0.0015716744 |\n",
      "| serial_timesteps   | 207360       |\n",
      "| time_elapsed       | 9.11e+03     |\n",
      "| total_timesteps    | 207360       |\n",
      "| value_loss         | 3.1303709    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0014179555 |\n",
      "| clipfrac           | 0.015625     |\n",
      "| explained_variance | 0.978        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 1630         |\n",
      "| policy_entropy     | 0.59142256   |\n",
      "| policy_loss        | -0.006596802 |\n",
      "| serial_timesteps   | 208640       |\n",
      "| time_elapsed       | 9.16e+03     |\n",
      "| total_timesteps    | 208640       |\n",
      "| value_loss         | 10.522829    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0173184    |\n",
      "| clipfrac           | 0.26953125   |\n",
      "| explained_variance | 0.965        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 1640         |\n",
      "| policy_entropy     | 1.2769002    |\n",
      "| policy_loss        | -0.018785603 |\n",
      "| serial_timesteps   | 209920       |\n",
      "| time_elapsed       | 9.2e+03      |\n",
      "| total_timesteps    | 209920       |\n",
      "| value_loss         | 1.3722603    |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.021455873  |\n",
      "| clipfrac           | 0.44726562   |\n",
      "| explained_variance | 0.908        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 1650         |\n",
      "| policy_entropy     | 1.0493002    |\n",
      "| policy_loss        | -0.027676985 |\n",
      "| serial_timesteps   | 211200       |\n",
      "| time_elapsed       | 9.25e+03     |\n",
      "| total_timesteps    | 211200       |\n",
      "| value_loss         | 3.4381578    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.020684373  |\n",
      "| clipfrac           | 0.23046875   |\n",
      "| explained_variance | 0.75         |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 1660         |\n",
      "| policy_entropy     | 0.9499921    |\n",
      "| policy_loss        | 0.0071590287 |\n",
      "| serial_timesteps   | 212480       |\n",
      "| time_elapsed       | 9.29e+03     |\n",
      "| total_timesteps    | 212480       |\n",
      "| value_loss         | 4.91061      |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.007035348 |\n",
      "| clipfrac           | 0.13085938  |\n",
      "| explained_variance | 0.957       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1670        |\n",
      "| policy_entropy     | 1.057699    |\n",
      "| policy_loss        | -0.008678   |\n",
      "| serial_timesteps   | 213760      |\n",
      "| time_elapsed       | 9.33e+03    |\n",
      "| total_timesteps    | 213760      |\n",
      "| value_loss         | 0.6575825   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.027728284  |\n",
      "| clipfrac           | 0.1796875    |\n",
      "| explained_variance | 0.708        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 1680         |\n",
      "| policy_entropy     | 0.76088715   |\n",
      "| policy_loss        | -0.039109457 |\n",
      "| serial_timesteps   | 215040       |\n",
      "| time_elapsed       | 9.38e+03     |\n",
      "| total_timesteps    | 215040       |\n",
      "| value_loss         | 33.34121     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.027232815  |\n",
      "| clipfrac           | 0.36914062   |\n",
      "| explained_variance | 0.0673       |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 1690         |\n",
      "| policy_entropy     | 0.64839494   |\n",
      "| policy_loss        | 0.0059676375 |\n",
      "| serial_timesteps   | 216320       |\n",
      "| time_elapsed       | 9.43e+03     |\n",
      "| total_timesteps    | 216320       |\n",
      "| value_loss         | 8.250156     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.025902048  |\n",
      "| clipfrac           | 0.29882812   |\n",
      "| explained_variance | 0.781        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 1700         |\n",
      "| policy_entropy     | 0.7897659    |\n",
      "| policy_loss        | -0.027235843 |\n",
      "| serial_timesteps   | 217600       |\n",
      "| time_elapsed       | 9.47e+03     |\n",
      "| total_timesteps    | 217600       |\n",
      "| value_loss         | 2.851893     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0118891075 |\n",
      "| clipfrac           | 0.15234375   |\n",
      "| explained_variance | 0.884        |\n",
      "| fps                | 21           |\n",
      "| n_updates          | 1710         |\n",
      "| policy_entropy     | 0.68193424   |\n",
      "| policy_loss        | -0.009242224 |\n",
      "| serial_timesteps   | 218880       |\n",
      "| time_elapsed       | 9.51e+03     |\n",
      "| total_timesteps    | 218880       |\n",
      "| value_loss         | 10.88446     |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.027630001   |\n",
      "| clipfrac           | 0.30078125    |\n",
      "| explained_variance | 0.995         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 1720          |\n",
      "| policy_entropy     | 0.6709598     |\n",
      "| policy_loss        | -0.0035555826 |\n",
      "| serial_timesteps   | 220160        |\n",
      "| time_elapsed       | 9.56e+03      |\n",
      "| total_timesteps    | 220160        |\n",
      "| value_loss         | 7.4477253     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.006143512  |\n",
      "| clipfrac           | 0.0859375    |\n",
      "| explained_variance | 0.965        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 1730         |\n",
      "| policy_entropy     | 0.5874299    |\n",
      "| policy_loss        | -0.010395797 |\n",
      "| serial_timesteps   | 221440       |\n",
      "| time_elapsed       | 9.61e+03     |\n",
      "| total_timesteps    | 221440       |\n",
      "| value_loss         | 0.74998164   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.012366251  |\n",
      "| clipfrac           | 0.087890625  |\n",
      "| explained_variance | 0.892        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 1740         |\n",
      "| policy_entropy     | 0.40238935   |\n",
      "| policy_loss        | -0.013908103 |\n",
      "| serial_timesteps   | 222720       |\n",
      "| time_elapsed       | 9.66e+03     |\n",
      "| total_timesteps    | 222720       |\n",
      "| value_loss         | 2.723348     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.004841213  |\n",
      "| clipfrac           | 0.05859375   |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 1750         |\n",
      "| policy_entropy     | 0.42654786   |\n",
      "| policy_loss        | 0.0030905362 |\n",
      "| serial_timesteps   | 224000       |\n",
      "| time_elapsed       | 9.7e+03      |\n",
      "| total_timesteps    | 224000       |\n",
      "| value_loss         | 2.0181732    |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=225000, episode_reward=499.94 +/- 1.92\n",
      "Episode length: 596.40 +/- 1.62\n",
      "------------------------------------\n",
      "| approxkl           | 0.037154578 |\n",
      "| clipfrac           | 0.26953125  |\n",
      "| explained_variance | 0.978       |\n",
      "| fps                | 31          |\n",
      "| n_updates          | 1760        |\n",
      "| policy_entropy     | 0.50821453  |\n",
      "| policy_loss        | 0.007527538 |\n",
      "| serial_timesteps   | 225280      |\n",
      "| time_elapsed       | 9.85e+03    |\n",
      "| total_timesteps    | 225280      |\n",
      "| value_loss         | 0.13131925  |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0033768564  |\n",
      "| clipfrac           | 0.025390625   |\n",
      "| explained_variance | 0.973         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 1770          |\n",
      "| policy_entropy     | 0.41182452    |\n",
      "| policy_loss        | -0.0023267395 |\n",
      "| serial_timesteps   | 226560        |\n",
      "| time_elapsed       | 9.9e+03       |\n",
      "| total_timesteps    | 226560        |\n",
      "| value_loss         | 2.174484      |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.017226715   |\n",
      "| clipfrac           | 0.09765625    |\n",
      "| explained_variance | 0.998         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 1780          |\n",
      "| policy_entropy     | 0.45823866    |\n",
      "| policy_loss        | -0.0035667527 |\n",
      "| serial_timesteps   | 227840        |\n",
      "| time_elapsed       | 9.95e+03      |\n",
      "| total_timesteps    | 227840        |\n",
      "| value_loss         | 2.9373093     |\n",
      "--------------------------------------\n",
      "-----------------------------------\n",
      "| approxkl           | 0.03663412 |\n",
      "| clipfrac           | 0.35742188 |\n",
      "| explained_variance | 0.975      |\n",
      "| fps                | 30         |\n",
      "| n_updates          | 1790       |\n",
      "| policy_entropy     | 0.5432243  |\n",
      "| policy_loss        | 0.0384751  |\n",
      "| serial_timesteps   | 229120     |\n",
      "| time_elapsed       | 1e+04      |\n",
      "| total_timesteps    | 229120     |\n",
      "| value_loss         | 0.6933925  |\n",
      "-----------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.02221206     |\n",
      "| clipfrac           | 0.09765625     |\n",
      "| explained_variance | 0.989          |\n",
      "| fps                | 23             |\n",
      "| n_updates          | 1800           |\n",
      "| policy_entropy     | 0.40784726     |\n",
      "| policy_loss        | -0.00089253206 |\n",
      "| serial_timesteps   | 230400         |\n",
      "| time_elapsed       | 1e+04          |\n",
      "| total_timesteps    | 230400         |\n",
      "| value_loss         | 0.6367008      |\n",
      "---------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.012834662  |\n",
      "| clipfrac           | 0.103515625  |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 1810         |\n",
      "| policy_entropy     | 0.83193487   |\n",
      "| policy_loss        | -0.011972907 |\n",
      "| serial_timesteps   | 231680       |\n",
      "| time_elapsed       | 1.01e+04     |\n",
      "| total_timesteps    | 231680       |\n",
      "| value_loss         | 0.838796     |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.02185878   |\n",
      "| clipfrac           | 0.30859375   |\n",
      "| explained_variance | 0.903        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 1820         |\n",
      "| policy_entropy     | 0.794566     |\n",
      "| policy_loss        | -0.020228483 |\n",
      "| serial_timesteps   | 232960       |\n",
      "| time_elapsed       | 1.01e+04     |\n",
      "| total_timesteps    | 232960       |\n",
      "| value_loss         | 0.26214346   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.035428114  |\n",
      "| clipfrac           | 0.076171875  |\n",
      "| explained_variance | 0.993        |\n",
      "| fps                | 25           |\n",
      "| n_updates          | 1830         |\n",
      "| policy_entropy     | 0.7529349    |\n",
      "| policy_loss        | -0.004178432 |\n",
      "| serial_timesteps   | 234240       |\n",
      "| time_elapsed       | 1.02e+04     |\n",
      "| total_timesteps    | 234240       |\n",
      "| value_loss         | 0.3624236    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.027220635  |\n",
      "| clipfrac           | 0.18164062   |\n",
      "| explained_variance | 0.957        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 1840         |\n",
      "| policy_entropy     | 0.5700375    |\n",
      "| policy_loss        | -0.006662092 |\n",
      "| serial_timesteps   | 235520       |\n",
      "| time_elapsed       | 1.02e+04     |\n",
      "| total_timesteps    | 235520       |\n",
      "| value_loss         | 0.8451224    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.041100167  |\n",
      "| clipfrac           | 0.38476562   |\n",
      "| explained_variance | 0.98         |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 1850         |\n",
      "| policy_entropy     | 0.9886812    |\n",
      "| policy_loss        | 0.0040893373 |\n",
      "| serial_timesteps   | 236800       |\n",
      "| time_elapsed       | 1.03e+04     |\n",
      "| total_timesteps    | 236800       |\n",
      "| value_loss         | 0.271153     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0035659513 |\n",
      "| clipfrac           | 0.05078125   |\n",
      "| explained_variance | 0.981        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 1860         |\n",
      "| policy_entropy     | 0.47879976   |\n",
      "| policy_loss        | -0.002156842 |\n",
      "| serial_timesteps   | 238080       |\n",
      "| time_elapsed       | 1.03e+04     |\n",
      "| total_timesteps    | 238080       |\n",
      "| value_loss         | 1.3580087    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.018409686   |\n",
      "| clipfrac           | 0.1796875     |\n",
      "| explained_variance | 0.997         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 1870          |\n",
      "| policy_entropy     | 0.61227083    |\n",
      "| policy_loss        | -0.0027182703 |\n",
      "| serial_timesteps   | 239360        |\n",
      "| time_elapsed       | 1.04e+04      |\n",
      "| total_timesteps    | 239360        |\n",
      "| value_loss         | 0.8112138     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.09758158  |\n",
      "| clipfrac           | 0.50390625  |\n",
      "| explained_variance | 0.95        |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1880        |\n",
      "| policy_entropy     | 0.4962515   |\n",
      "| policy_loss        | 0.039321825 |\n",
      "| serial_timesteps   | 240640      |\n",
      "| time_elapsed       | 1.04e+04    |\n",
      "| total_timesteps    | 240640      |\n",
      "| value_loss         | 0.50890917  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.004947719  |\n",
      "| clipfrac           | 0.044921875  |\n",
      "| explained_variance | 0.97         |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 1890         |\n",
      "| policy_entropy     | 0.3040374    |\n",
      "| policy_loss        | 0.0009446129 |\n",
      "| serial_timesteps   | 241920       |\n",
      "| time_elapsed       | 1.05e+04     |\n",
      "| total_timesteps    | 241920       |\n",
      "| value_loss         | 0.9028504    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.007967027 |\n",
      "| clipfrac           | 0.11328125  |\n",
      "| explained_variance | 0.997       |\n",
      "| fps                | 28          |\n",
      "| n_updates          | 1900        |\n",
      "| policy_entropy     | 0.72010416  |\n",
      "| policy_loss        | -0.01087328 |\n",
      "| serial_timesteps   | 243200      |\n",
      "| time_elapsed       | 1.05e+04    |\n",
      "| total_timesteps    | 243200      |\n",
      "| value_loss         | 1.6572571   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.010657905   |\n",
      "| clipfrac           | 0.10546875    |\n",
      "| explained_variance | 0.571         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 1910          |\n",
      "| policy_entropy     | 0.7951854     |\n",
      "| policy_loss        | -0.0026767026 |\n",
      "| serial_timesteps   | 244480        |\n",
      "| time_elapsed       | 1.06e+04      |\n",
      "| total_timesteps    | 244480        |\n",
      "| value_loss         | 1.3224988     |\n",
      "--------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.0026176637   |\n",
      "| clipfrac           | 0.025390625    |\n",
      "| explained_variance | 0.986          |\n",
      "| fps                | 25             |\n",
      "| n_updates          | 1920           |\n",
      "| policy_entropy     | 0.5327015      |\n",
      "| policy_loss        | -0.00033787882 |\n",
      "| serial_timesteps   | 245760         |\n",
      "| time_elapsed       | 1.06e+04       |\n",
      "| total_timesteps    | 245760         |\n",
      "| value_loss         | 0.5821571      |\n",
      "---------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.012882082  |\n",
      "| clipfrac           | 0.10546875   |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 1930         |\n",
      "| policy_entropy     | 0.97968173   |\n",
      "| policy_loss        | -0.018080967 |\n",
      "| serial_timesteps   | 247040       |\n",
      "| time_elapsed       | 1.07e+04     |\n",
      "| total_timesteps    | 247040       |\n",
      "| value_loss         | 2.2484245    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.013815007   |\n",
      "| clipfrac           | 0.017578125   |\n",
      "| explained_variance | 0.909         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 1940          |\n",
      "| policy_entropy     | 0.5360431     |\n",
      "| policy_loss        | -0.0068434253 |\n",
      "| serial_timesteps   | 248320        |\n",
      "| time_elapsed       | 1.07e+04      |\n",
      "| total_timesteps    | 248320        |\n",
      "| value_loss         | 0.28301305    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.002497869  |\n",
      "| clipfrac           | 0.029296875  |\n",
      "| explained_variance | -1.12        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 1950         |\n",
      "| policy_entropy     | 0.2668785    |\n",
      "| policy_loss        | -0.008677173 |\n",
      "| serial_timesteps   | 249600       |\n",
      "| time_elapsed       | 1.07e+04     |\n",
      "| total_timesteps    | 249600       |\n",
      "| value_loss         | 6.5914807    |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=250000, episode_reward=540.32 +/- 9.41\n",
      "Episode length: 585.00 +/- 4.15\n",
      "------------------------------------\n",
      "| approxkl           | 0.025789483 |\n",
      "| clipfrac           | 0.21875     |\n",
      "| explained_variance | 0.98        |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 1960        |\n",
      "| policy_entropy     | 0.4761696   |\n",
      "| policy_loss        | 0.01589824  |\n",
      "| serial_timesteps   | 250880      |\n",
      "| time_elapsed       | 1.09e+04    |\n",
      "| total_timesteps    | 250880      |\n",
      "| value_loss         | 0.15089436  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.06783161  |\n",
      "| clipfrac           | 0.26953125  |\n",
      "| explained_variance | 0.984       |\n",
      "| fps                | 22          |\n",
      "| n_updates          | 1970        |\n",
      "| policy_entropy     | 0.84243923  |\n",
      "| policy_loss        | 0.002495393 |\n",
      "| serial_timesteps   | 252160      |\n",
      "| time_elapsed       | 1.09e+04    |\n",
      "| total_timesteps    | 252160      |\n",
      "| value_loss         | 6.802767    |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.024448058  |\n",
      "| clipfrac           | 0.27148438   |\n",
      "| explained_variance | 0.797        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 1980         |\n",
      "| policy_entropy     | 1.1131953    |\n",
      "| policy_loss        | 0.0033805117 |\n",
      "| serial_timesteps   | 253440       |\n",
      "| time_elapsed       | 1.1e+04      |\n",
      "| total_timesteps    | 253440       |\n",
      "| value_loss         | 0.7221013    |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.014427273  |\n",
      "| clipfrac           | 0.140625     |\n",
      "| explained_variance | 0.983        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 1990         |\n",
      "| policy_entropy     | 0.9336724    |\n",
      "| policy_loss        | -0.007177237 |\n",
      "| serial_timesteps   | 254720       |\n",
      "| time_elapsed       | 1.1e+04      |\n",
      "| total_timesteps    | 254720       |\n",
      "| value_loss         | 5.828614     |\n",
      "-------------------------------------\n",
      "-----------------------------------\n",
      "| approxkl           | 0.04793697 |\n",
      "| clipfrac           | 0.34960938 |\n",
      "| explained_variance | 0.88       |\n",
      "| fps                | 30         |\n",
      "| n_updates          | 2000       |\n",
      "| policy_entropy     | 0.84103435 |\n",
      "| policy_loss        | 0.01559207 |\n",
      "| serial_timesteps   | 256000     |\n",
      "| time_elapsed       | 1.11e+04   |\n",
      "| total_timesteps    | 256000     |\n",
      "| value_loss         | 0.21277203 |\n",
      "-----------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.043132417 |\n",
      "| clipfrac           | 0.30078125  |\n",
      "| explained_variance | 0.756       |\n",
      "| fps                | 28          |\n",
      "| n_updates          | 2010        |\n",
      "| policy_entropy     | 0.84399766  |\n",
      "| policy_loss        | 0.009595007 |\n",
      "| serial_timesteps   | 257280      |\n",
      "| time_elapsed       | 1.11e+04    |\n",
      "| total_timesteps    | 257280      |\n",
      "| value_loss         | 1.4628537   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.02420723   |\n",
      "| clipfrac           | 0.21679688   |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 2020         |\n",
      "| policy_entropy     | 1.0762655    |\n",
      "| policy_loss        | -0.023633156 |\n",
      "| serial_timesteps   | 258560       |\n",
      "| time_elapsed       | 1.12e+04     |\n",
      "| total_timesteps    | 258560       |\n",
      "| value_loss         | 0.771272     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.029196152  |\n",
      "| clipfrac           | 0.265625     |\n",
      "| explained_variance | 0.982        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 2030         |\n",
      "| policy_entropy     | 0.5573954    |\n",
      "| policy_loss        | -0.018532433 |\n",
      "| serial_timesteps   | 259840       |\n",
      "| time_elapsed       | 1.12e+04     |\n",
      "| total_timesteps    | 259840       |\n",
      "| value_loss         | 0.17824537   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.007273634   |\n",
      "| clipfrac           | 0.05859375    |\n",
      "| explained_variance | 0.959         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 2040          |\n",
      "| policy_entropy     | 0.5287995     |\n",
      "| policy_loss        | -0.0059963036 |\n",
      "| serial_timesteps   | 261120        |\n",
      "| time_elapsed       | 1.13e+04      |\n",
      "| total_timesteps    | 261120        |\n",
      "| value_loss         | 0.37800157    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0066048554 |\n",
      "| clipfrac           | 0.083984375  |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 2050         |\n",
      "| policy_entropy     | 0.5989459    |\n",
      "| policy_loss        | -0.006257032 |\n",
      "| serial_timesteps   | 262400       |\n",
      "| time_elapsed       | 1.13e+04     |\n",
      "| total_timesteps    | 262400       |\n",
      "| value_loss         | 0.49505997   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0056194537 |\n",
      "| clipfrac           | 0.083984375  |\n",
      "| explained_variance | 0.995        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 2060         |\n",
      "| policy_entropy     | 0.6954724    |\n",
      "| policy_loss        | 0.007487735  |\n",
      "| serial_timesteps   | 263680       |\n",
      "| time_elapsed       | 1.14e+04     |\n",
      "| total_timesteps    | 263680       |\n",
      "| value_loss         | 0.75603837   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.003519549   |\n",
      "| clipfrac           | 0.041015625   |\n",
      "| explained_variance | 0.947         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 2070          |\n",
      "| policy_entropy     | 0.43114       |\n",
      "| policy_loss        | 0.00037795084 |\n",
      "| serial_timesteps   | 264960        |\n",
      "| time_elapsed       | 1.14e+04      |\n",
      "| total_timesteps    | 264960        |\n",
      "| value_loss         | 0.78088987    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.008562599   |\n",
      "| clipfrac           | 0.078125      |\n",
      "| explained_variance | 0.995         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 2080          |\n",
      "| policy_entropy     | 0.35286358    |\n",
      "| policy_loss        | -0.0007519949 |\n",
      "| serial_timesteps   | 266240        |\n",
      "| time_elapsed       | 1.15e+04      |\n",
      "| total_timesteps    | 266240        |\n",
      "| value_loss         | 0.39324492    |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.004409517 |\n",
      "| clipfrac           | 0.05859375  |\n",
      "| explained_variance | 0.999       |\n",
      "| fps                | 28          |\n",
      "| n_updates          | 2090        |\n",
      "| policy_entropy     | 0.5216995   |\n",
      "| policy_loss        | 0.004991113 |\n",
      "| serial_timesteps   | 267520      |\n",
      "| time_elapsed       | 1.15e+04    |\n",
      "| total_timesteps    | 267520      |\n",
      "| value_loss         | 0.84917974  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.012545493  |\n",
      "| clipfrac           | 0.17382812   |\n",
      "| explained_variance | 0.624        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 2100         |\n",
      "| policy_entropy     | 0.48047292   |\n",
      "| policy_loss        | 0.0042341705 |\n",
      "| serial_timesteps   | 268800       |\n",
      "| time_elapsed       | 1.16e+04     |\n",
      "| total_timesteps    | 268800       |\n",
      "| value_loss         | 2.03463      |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.010665629   |\n",
      "| clipfrac           | 0.107421875   |\n",
      "| explained_variance | 0.956         |\n",
      "| fps                | 24            |\n",
      "| n_updates          | 2110          |\n",
      "| policy_entropy     | 0.39685175    |\n",
      "| policy_loss        | -0.0036001904 |\n",
      "| serial_timesteps   | 270080        |\n",
      "| time_elapsed       | 1.16e+04      |\n",
      "| total_timesteps    | 270080        |\n",
      "| value_loss         | 1.9995906     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0033626978 |\n",
      "| clipfrac           | 0.04296875   |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 2120         |\n",
      "| policy_entropy     | 0.44082406   |\n",
      "| policy_loss        | -0.009051447 |\n",
      "| serial_timesteps   | 271360       |\n",
      "| time_elapsed       | 1.16e+04     |\n",
      "| total_timesteps    | 271360       |\n",
      "| value_loss         | 2.3341236    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.03936207  |\n",
      "| clipfrac           | 0.32421875  |\n",
      "| explained_variance | 0.928       |\n",
      "| fps                | 31          |\n",
      "| n_updates          | 2130        |\n",
      "| policy_entropy     | 0.52351683  |\n",
      "| policy_loss        | 0.032367244 |\n",
      "| serial_timesteps   | 272640      |\n",
      "| time_elapsed       | 1.17e+04    |\n",
      "| total_timesteps    | 272640      |\n",
      "| value_loss         | 0.2827704   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.019310025 |\n",
      "| clipfrac           | 0.23242188  |\n",
      "| explained_variance | 0.982       |\n",
      "| fps                | 25          |\n",
      "| n_updates          | 2140        |\n",
      "| policy_entropy     | 0.4080375   |\n",
      "| policy_loss        | 0.003986274 |\n",
      "| serial_timesteps   | 273920      |\n",
      "| time_elapsed       | 1.17e+04    |\n",
      "| total_timesteps    | 273920      |\n",
      "| value_loss         | 0.39631423  |\n",
      "------------------------------------\n",
      "Eval num_timesteps=275000, episode_reward=537.20 +/- 7.13\n",
      "Episode length: 585.20 +/- 1.17\n",
      "-------------------------------------\n",
      "| approxkl           | 0.00532661   |\n",
      "| clipfrac           | 0.0546875    |\n",
      "| explained_variance | 0.228        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 2150         |\n",
      "| policy_entropy     | 0.42147982   |\n",
      "| policy_loss        | -0.010396456 |\n",
      "| serial_timesteps   | 275200       |\n",
      "| time_elapsed       | 1.19e+04     |\n",
      "| total_timesteps    | 275200       |\n",
      "| value_loss         | 0.711895     |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.003210299  |\n",
      "| clipfrac           | 0.041015625  |\n",
      "| explained_variance | 0.995        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 2160         |\n",
      "| policy_entropy     | 0.41311544   |\n",
      "| policy_loss        | 0.0029935115 |\n",
      "| serial_timesteps   | 276480       |\n",
      "| time_elapsed       | 1.19e+04     |\n",
      "| total_timesteps    | 276480       |\n",
      "| value_loss         | 0.46508247   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.021563774  |\n",
      "| clipfrac           | 0.23242188   |\n",
      "| explained_variance | 0.986        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 2170         |\n",
      "| policy_entropy     | 0.8622347    |\n",
      "| policy_loss        | -0.008650607 |\n",
      "| serial_timesteps   | 277760       |\n",
      "| time_elapsed       | 1.2e+04      |\n",
      "| total_timesteps    | 277760       |\n",
      "| value_loss         | 5.1688905    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0019197091  |\n",
      "| clipfrac           | 0.01953125    |\n",
      "| explained_variance | 0.654         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 2180          |\n",
      "| policy_entropy     | 0.45278555    |\n",
      "| policy_loss        | -0.0023504011 |\n",
      "| serial_timesteps   | 279040        |\n",
      "| time_elapsed       | 1.2e+04       |\n",
      "| total_timesteps    | 279040        |\n",
      "| value_loss         | 0.4835814     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.033608973 |\n",
      "| clipfrac           | 0.22851562  |\n",
      "| explained_variance | 0.983       |\n",
      "| fps                | 27          |\n",
      "| n_updates          | 2190        |\n",
      "| policy_entropy     | 0.3341057   |\n",
      "| policy_loss        | 0.017865729 |\n",
      "| serial_timesteps   | 280320      |\n",
      "| time_elapsed       | 1.21e+04    |\n",
      "| total_timesteps    | 280320      |\n",
      "| value_loss         | 0.72282624  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.023844548  |\n",
      "| clipfrac           | 0.15429688   |\n",
      "| explained_variance | 0.0353       |\n",
      "| fps                | 21           |\n",
      "| n_updates          | 2200         |\n",
      "| policy_entropy     | 0.87206054   |\n",
      "| policy_loss        | -0.017736675 |\n",
      "| serial_timesteps   | 281600       |\n",
      "| time_elapsed       | 1.21e+04     |\n",
      "| total_timesteps    | 281600       |\n",
      "| value_loss         | 16.539637    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.010052502  |\n",
      "| clipfrac           | 0.15234375   |\n",
      "| explained_variance | 0.952        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 2210         |\n",
      "| policy_entropy     | 0.8133383    |\n",
      "| policy_loss        | 0.0014731071 |\n",
      "| serial_timesteps   | 282880       |\n",
      "| time_elapsed       | 1.22e+04     |\n",
      "| total_timesteps    | 282880       |\n",
      "| value_loss         | 0.7209681    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0034620597 |\n",
      "| clipfrac           | 0.015625     |\n",
      "| explained_variance | 0.924        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 2220         |\n",
      "| policy_entropy     | 0.5794964    |\n",
      "| policy_loss        | 0.0023317365 |\n",
      "| serial_timesteps   | 284160       |\n",
      "| time_elapsed       | 1.22e+04     |\n",
      "| total_timesteps    | 284160       |\n",
      "| value_loss         | 4.7344146    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.021772893  |\n",
      "| clipfrac           | 0.20117188   |\n",
      "| explained_variance | 0.984        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 2230         |\n",
      "| policy_entropy     | 0.63826793   |\n",
      "| policy_loss        | -0.015514887 |\n",
      "| serial_timesteps   | 285440       |\n",
      "| time_elapsed       | 1.23e+04     |\n",
      "| total_timesteps    | 285440       |\n",
      "| value_loss         | 17.210539    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0039210836 |\n",
      "| clipfrac           | 0.03515625   |\n",
      "| explained_variance | 0.923        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 2240         |\n",
      "| policy_entropy     | 0.5842863    |\n",
      "| policy_loss        | 0.0030982576 |\n",
      "| serial_timesteps   | 286720       |\n",
      "| time_elapsed       | 1.23e+04     |\n",
      "| total_timesteps    | 286720       |\n",
      "| value_loss         | 1.4985999    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.018496605  |\n",
      "| clipfrac           | 0.14648438   |\n",
      "| explained_variance | 0.91         |\n",
      "| fps                | 21           |\n",
      "| n_updates          | 2250         |\n",
      "| policy_entropy     | 0.5843568    |\n",
      "| policy_loss        | -0.013480016 |\n",
      "| serial_timesteps   | 288000       |\n",
      "| time_elapsed       | 1.23e+04     |\n",
      "| total_timesteps    | 288000       |\n",
      "| value_loss         | 9.855389     |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.03266868  |\n",
      "| clipfrac           | 0.13085938  |\n",
      "| explained_variance | 0.989       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 2260        |\n",
      "| policy_entropy     | 0.41207135  |\n",
      "| policy_loss        | -0.00292911 |\n",
      "| serial_timesteps   | 289280      |\n",
      "| time_elapsed       | 1.24e+04    |\n",
      "| total_timesteps    | 289280      |\n",
      "| value_loss         | 0.6712372   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.01737301    |\n",
      "| clipfrac           | 0.203125      |\n",
      "| explained_variance | 0.89          |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 2270          |\n",
      "| policy_entropy     | 0.4849792     |\n",
      "| policy_loss        | -0.0056732963 |\n",
      "| serial_timesteps   | 290560        |\n",
      "| time_elapsed       | 1.24e+04      |\n",
      "| total_timesteps    | 290560        |\n",
      "| value_loss         | 2.0794334     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.011748182   |\n",
      "| clipfrac           | 0.14648438    |\n",
      "| explained_variance | 0.937         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 2280          |\n",
      "| policy_entropy     | 0.55526775    |\n",
      "| policy_loss        | -0.0018414378 |\n",
      "| serial_timesteps   | 291840        |\n",
      "| time_elapsed       | 1.25e+04      |\n",
      "| total_timesteps    | 291840        |\n",
      "| value_loss         | 3.8726587     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.008063468 |\n",
      "| clipfrac           | 0.09765625  |\n",
      "| explained_variance | 0.999       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 2290        |\n",
      "| policy_entropy     | 0.37933242  |\n",
      "| policy_loss        | 0.002496386 |\n",
      "| serial_timesteps   | 293120      |\n",
      "| time_elapsed       | 1.25e+04    |\n",
      "| total_timesteps    | 293120      |\n",
      "| value_loss         | 0.4084357   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.010337021  |\n",
      "| clipfrac           | 0.09765625   |\n",
      "| explained_variance | 0.96         |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 2300         |\n",
      "| policy_entropy     | 0.8189586    |\n",
      "| policy_loss        | 0.0047215926 |\n",
      "| serial_timesteps   | 294400       |\n",
      "| time_elapsed       | 1.26e+04     |\n",
      "| total_timesteps    | 294400       |\n",
      "| value_loss         | 0.5234267    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.013181358  |\n",
      "| clipfrac           | 0.15625      |\n",
      "| explained_variance | 0.988        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 2310         |\n",
      "| policy_entropy     | 0.7173063    |\n",
      "| policy_loss        | -0.016794242 |\n",
      "| serial_timesteps   | 295680       |\n",
      "| time_elapsed       | 1.26e+04     |\n",
      "| total_timesteps    | 295680       |\n",
      "| value_loss         | 0.6401084    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.016471358  |\n",
      "| clipfrac           | 0.13867188   |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 2320         |\n",
      "| policy_entropy     | 0.48649895   |\n",
      "| policy_loss        | 0.0054748827 |\n",
      "| serial_timesteps   | 296960       |\n",
      "| time_elapsed       | 1.27e+04     |\n",
      "| total_timesteps    | 296960       |\n",
      "| value_loss         | 0.56987935   |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------\n",
      "| approxkl           | 0.0682412   |\n",
      "| clipfrac           | 0.5449219   |\n",
      "| explained_variance | 0.531       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 2330        |\n",
      "| policy_entropy     | 1.0255315   |\n",
      "| policy_loss        | 0.008082462 |\n",
      "| serial_timesteps   | 298240      |\n",
      "| time_elapsed       | 1.27e+04    |\n",
      "| total_timesteps    | 298240      |\n",
      "| value_loss         | 3.6643345   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.023681942  |\n",
      "| clipfrac           | 0.19726562   |\n",
      "| explained_variance | 0.987        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 2340         |\n",
      "| policy_entropy     | 0.69434696   |\n",
      "| policy_loss        | -0.013103076 |\n",
      "| serial_timesteps   | 299520       |\n",
      "| time_elapsed       | 1.28e+04     |\n",
      "| total_timesteps    | 299520       |\n",
      "| value_loss         | 1.6995772    |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=300000, episode_reward=525.50 +/- 1.70\n",
      "Episode length: 584.20 +/- 1.17\n",
      "-------------------------------------\n",
      "| approxkl           | 0.008312795  |\n",
      "| clipfrac           | 0.14453125   |\n",
      "| explained_variance | 0.98         |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 2350         |\n",
      "| policy_entropy     | 0.55983347   |\n",
      "| policy_loss        | 0.0006820299 |\n",
      "| serial_timesteps   | 300800       |\n",
      "| time_elapsed       | 1.29e+04     |\n",
      "| total_timesteps    | 300800       |\n",
      "| value_loss         | 0.20803761   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0125334095 |\n",
      "| clipfrac           | 0.1484375    |\n",
      "| explained_variance | 0.988        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 2360         |\n",
      "| policy_entropy     | 0.5837317    |\n",
      "| policy_loss        | -0.011591485 |\n",
      "| serial_timesteps   | 302080       |\n",
      "| time_elapsed       | 1.3e+04      |\n",
      "| total_timesteps    | 302080       |\n",
      "| value_loss         | 0.41110304   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.020328613   |\n",
      "| clipfrac           | 0.203125      |\n",
      "| explained_variance | 0.996         |\n",
      "| fps                | 26            |\n",
      "| n_updates          | 2370          |\n",
      "| policy_entropy     | 0.5559931     |\n",
      "| policy_loss        | -0.0054202923 |\n",
      "| serial_timesteps   | 303360        |\n",
      "| time_elapsed       | 1.3e+04       |\n",
      "| total_timesteps    | 303360        |\n",
      "| value_loss         | 1.7343104     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.017656125 |\n",
      "| clipfrac           | 0.20703125  |\n",
      "| explained_variance | 0.841       |\n",
      "| fps                | 31          |\n",
      "| n_updates          | 2380        |\n",
      "| policy_entropy     | 0.84263206  |\n",
      "| policy_loss        | -0.00873373 |\n",
      "| serial_timesteps   | 304640      |\n",
      "| time_elapsed       | 1.31e+04    |\n",
      "| total_timesteps    | 304640      |\n",
      "| value_loss         | 0.15474464  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.010834111  |\n",
      "| clipfrac           | 0.1796875    |\n",
      "| explained_variance | 0.994        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 2390         |\n",
      "| policy_entropy     | 0.76001555   |\n",
      "| policy_loss        | -0.010881519 |\n",
      "| serial_timesteps   | 305920       |\n",
      "| time_elapsed       | 1.31e+04     |\n",
      "| total_timesteps    | 305920       |\n",
      "| value_loss         | 0.15787521   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0154896     |\n",
      "| clipfrac           | 0.15234375    |\n",
      "| explained_variance | 0.997         |\n",
      "| fps                | 24            |\n",
      "| n_updates          | 2400          |\n",
      "| policy_entropy     | 0.5722726     |\n",
      "| policy_loss        | -0.0037665344 |\n",
      "| serial_timesteps   | 307200        |\n",
      "| time_elapsed       | 1.31e+04      |\n",
      "| total_timesteps    | 307200        |\n",
      "| value_loss         | 0.9363105     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.017570801  |\n",
      "| clipfrac           | 0.26367188   |\n",
      "| explained_variance | 0.768        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 2410         |\n",
      "| policy_entropy     | 0.8082138    |\n",
      "| policy_loss        | -0.016776104 |\n",
      "| serial_timesteps   | 308480       |\n",
      "| time_elapsed       | 1.32e+04     |\n",
      "| total_timesteps    | 308480       |\n",
      "| value_loss         | 0.17070073   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.02117928   |\n",
      "| clipfrac           | 0.23242188   |\n",
      "| explained_variance | 0.985        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 2420         |\n",
      "| policy_entropy     | 1.0231346    |\n",
      "| policy_loss        | -0.002426526 |\n",
      "| serial_timesteps   | 309760       |\n",
      "| time_elapsed       | 1.32e+04     |\n",
      "| total_timesteps    | 309760       |\n",
      "| value_loss         | 0.41285926   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.054147184  |\n",
      "| clipfrac           | 0.30078125   |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 2430         |\n",
      "| policy_entropy     | 0.67983156   |\n",
      "| policy_loss        | -0.023761613 |\n",
      "| serial_timesteps   | 311040       |\n",
      "| time_elapsed       | 1.33e+04     |\n",
      "| total_timesteps    | 311040       |\n",
      "| value_loss         | 0.3060713    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.047239944 |\n",
      "| clipfrac           | 0.40625     |\n",
      "| explained_variance | 0.98        |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 2440        |\n",
      "| policy_entropy     | 0.5662927   |\n",
      "| policy_loss        | 0.042501066 |\n",
      "| serial_timesteps   | 312320      |\n",
      "| time_elapsed       | 1.33e+04    |\n",
      "| total_timesteps    | 312320      |\n",
      "| value_loss         | 0.41305602  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.00925733   |\n",
      "| clipfrac           | 0.11328125   |\n",
      "| explained_variance | 0.922        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 2450         |\n",
      "| policy_entropy     | 0.4958525    |\n",
      "| policy_loss        | -0.002569885 |\n",
      "| serial_timesteps   | 313600       |\n",
      "| time_elapsed       | 1.34e+04     |\n",
      "| total_timesteps    | 313600       |\n",
      "| value_loss         | 13.639899    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.020794734 |\n",
      "| clipfrac           | 0.140625    |\n",
      "| explained_variance | 0.993       |\n",
      "| fps                | 24          |\n",
      "| n_updates          | 2460        |\n",
      "| policy_entropy     | 0.49623513  |\n",
      "| policy_loss        | -0.02554473 |\n",
      "| serial_timesteps   | 314880      |\n",
      "| time_elapsed       | 1.34e+04    |\n",
      "| total_timesteps    | 314880      |\n",
      "| value_loss         | 1.0855234   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0037811606 |\n",
      "| clipfrac           | 0.033203125  |\n",
      "| explained_variance | 0.953        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 2470         |\n",
      "| policy_entropy     | 0.5249722    |\n",
      "| policy_loss        | -0.004473174 |\n",
      "| serial_timesteps   | 316160       |\n",
      "| time_elapsed       | 1.35e+04     |\n",
      "| total_timesteps    | 316160       |\n",
      "| value_loss         | 1.5092552    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.020677105  |\n",
      "| clipfrac           | 0.16601562   |\n",
      "| explained_variance | 0.974        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 2480         |\n",
      "| policy_entropy     | 0.64034694   |\n",
      "| policy_loss        | -0.003219769 |\n",
      "| serial_timesteps   | 317440       |\n",
      "| time_elapsed       | 1.35e+04     |\n",
      "| total_timesteps    | 317440       |\n",
      "| value_loss         | 5.590628     |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.024816442   |\n",
      "| clipfrac           | 0.16601562    |\n",
      "| explained_variance | 0.827         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 2490          |\n",
      "| policy_entropy     | 1.0493623     |\n",
      "| policy_loss        | -0.0073555717 |\n",
      "| serial_timesteps   | 318720        |\n",
      "| time_elapsed       | 1.36e+04      |\n",
      "| total_timesteps    | 318720        |\n",
      "| value_loss         | 2.1162782     |\n",
      "--------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.027924348  |\n",
      "| clipfrac           | 0.23828125   |\n",
      "| explained_variance | 0.759        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 2500         |\n",
      "| policy_entropy     | 0.8449948    |\n",
      "| policy_loss        | 0.0037702445 |\n",
      "| serial_timesteps   | 320000       |\n",
      "| time_elapsed       | 1.36e+04     |\n",
      "| total_timesteps    | 320000       |\n",
      "| value_loss         | 0.74762183   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.022111028   |\n",
      "| clipfrac           | 0.125         |\n",
      "| explained_variance | 0.971         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 2510          |\n",
      "| policy_entropy     | 0.6081834     |\n",
      "| policy_loss        | -0.0066766986 |\n",
      "| serial_timesteps   | 321280        |\n",
      "| time_elapsed       | 1.37e+04      |\n",
      "| total_timesteps    | 321280        |\n",
      "| value_loss         | 1.0851252     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.013602332 |\n",
      "| clipfrac           | 0.22460938  |\n",
      "| explained_variance | 0.999       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 2520        |\n",
      "| policy_entropy     | 0.81494147  |\n",
      "| policy_loss        | -0.01947174 |\n",
      "| serial_timesteps   | 322560      |\n",
      "| time_elapsed       | 1.37e+04    |\n",
      "| total_timesteps    | 322560      |\n",
      "| value_loss         | 0.5447535   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.00948688   |\n",
      "| clipfrac           | 0.18164062   |\n",
      "| explained_variance | 0.908        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 2530         |\n",
      "| policy_entropy     | 0.96181434   |\n",
      "| policy_loss        | -0.006903183 |\n",
      "| serial_timesteps   | 323840       |\n",
      "| time_elapsed       | 1.38e+04     |\n",
      "| total_timesteps    | 323840       |\n",
      "| value_loss         | 0.18707478   |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=325000, episode_reward=525.40 +/- 3.07\n",
      "Episode length: 587.00 +/- 0.63\n",
      "--------------------------------------\n",
      "| approxkl           | 0.028781379   |\n",
      "| clipfrac           | 0.15625       |\n",
      "| explained_variance | 0.893         |\n",
      "| fps                | 1             |\n",
      "| n_updates          | 2540          |\n",
      "| policy_entropy     | 0.61426526    |\n",
      "| policy_loss        | -0.0028269917 |\n",
      "| serial_timesteps   | 325120        |\n",
      "| time_elapsed       | 1.38e+04      |\n",
      "| total_timesteps    | 325120        |\n",
      "| value_loss         | 4.095084      |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.010865228   |\n",
      "| clipfrac           | 0.107421875   |\n",
      "| explained_variance | 0.877         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 2550          |\n",
      "| policy_entropy     | 0.81023353    |\n",
      "| policy_loss        | -0.0056676418 |\n",
      "| serial_timesteps   | 326400        |\n",
      "| time_elapsed       | 1.4e+04       |\n",
      "| total_timesteps    | 326400        |\n",
      "| value_loss         | 0.19241093    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.01624818   |\n",
      "| clipfrac           | 0.2265625    |\n",
      "| explained_variance | 0.992        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 2560         |\n",
      "| policy_entropy     | 0.9506034    |\n",
      "| policy_loss        | -0.011049179 |\n",
      "| serial_timesteps   | 327680       |\n",
      "| time_elapsed       | 1.4e+04      |\n",
      "| total_timesteps    | 327680       |\n",
      "| value_loss         | 0.966777     |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.00978696    |\n",
      "| clipfrac           | 0.15039062    |\n",
      "| explained_variance | 0.997         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 2570          |\n",
      "| policy_entropy     | 0.8873728     |\n",
      "| policy_loss        | -0.0055922195 |\n",
      "| serial_timesteps   | 328960        |\n",
      "| time_elapsed       | 1.41e+04      |\n",
      "| total_timesteps    | 328960        |\n",
      "| value_loss         | 1.4066778     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.032529168   |\n",
      "| clipfrac           | 0.3359375     |\n",
      "| explained_variance | 0.91          |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 2580          |\n",
      "| policy_entropy     | 1.1077974     |\n",
      "| policy_loss        | -0.0062700766 |\n",
      "| serial_timesteps   | 330240        |\n",
      "| time_elapsed       | 1.41e+04      |\n",
      "| total_timesteps    | 330240        |\n",
      "| value_loss         | 0.42070502    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.031840336  |\n",
      "| clipfrac           | 0.25390625   |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 2590         |\n",
      "| policy_entropy     | 0.9563528    |\n",
      "| policy_loss        | -0.016272604 |\n",
      "| serial_timesteps   | 331520       |\n",
      "| time_elapsed       | 1.41e+04     |\n",
      "| total_timesteps    | 331520       |\n",
      "| value_loss         | 0.5463805    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.014908862   |\n",
      "| clipfrac           | 0.17578125    |\n",
      "| explained_variance | 0.991         |\n",
      "| fps                | 27            |\n",
      "| n_updates          | 2600          |\n",
      "| policy_entropy     | 0.5766751     |\n",
      "| policy_loss        | -0.0045033926 |\n",
      "| serial_timesteps   | 332800        |\n",
      "| time_elapsed       | 1.42e+04      |\n",
      "| total_timesteps    | 332800        |\n",
      "| value_loss         | 3.724705      |\n",
      "--------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.032030962    |\n",
      "| clipfrac           | 0.29101562     |\n",
      "| explained_variance | 0.741          |\n",
      "| fps                | 31             |\n",
      "| n_updates          | 2610           |\n",
      "| policy_entropy     | 0.6346405      |\n",
      "| policy_loss        | -0.00055348966 |\n",
      "| serial_timesteps   | 334080         |\n",
      "| time_elapsed       | 1.42e+04       |\n",
      "| total_timesteps    | 334080         |\n",
      "| value_loss         | 0.7550593      |\n",
      "---------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0077576963 |\n",
      "| clipfrac           | 0.091796875  |\n",
      "| explained_variance | 0.632        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 2620         |\n",
      "| policy_entropy     | 0.8166324    |\n",
      "| policy_loss        | -0.009056894 |\n",
      "| serial_timesteps   | 335360       |\n",
      "| time_elapsed       | 1.43e+04     |\n",
      "| total_timesteps    | 335360       |\n",
      "| value_loss         | 4.496577     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.041207094  |\n",
      "| clipfrac           | 0.22460938   |\n",
      "| explained_variance | 0.994        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 2630         |\n",
      "| policy_entropy     | 0.7351307    |\n",
      "| policy_loss        | -0.014197291 |\n",
      "| serial_timesteps   | 336640       |\n",
      "| time_elapsed       | 1.43e+04     |\n",
      "| total_timesteps    | 336640       |\n",
      "| value_loss         | 1.6117661    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.009818468 |\n",
      "| clipfrac           | 0.123046875 |\n",
      "| explained_variance | 0.888       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 2640        |\n",
      "| policy_entropy     | 0.6775207   |\n",
      "| policy_loss        | 0.002239476 |\n",
      "| serial_timesteps   | 337920      |\n",
      "| time_elapsed       | 1.44e+04    |\n",
      "| total_timesteps    | 337920      |\n",
      "| value_loss         | 0.66329426  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.007109629  |\n",
      "| clipfrac           | 0.09765625   |\n",
      "| explained_variance | 0.561        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 2650         |\n",
      "| policy_entropy     | 0.6290936    |\n",
      "| policy_loss        | 9.644753e-05 |\n",
      "| serial_timesteps   | 339200       |\n",
      "| time_elapsed       | 1.44e+04     |\n",
      "| total_timesteps    | 339200       |\n",
      "| value_loss         | 3.9480329    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.004955259 |\n",
      "| clipfrac           | 0.080078125 |\n",
      "| explained_variance | 0.998       |\n",
      "| fps                | 22          |\n",
      "| n_updates          | 2660        |\n",
      "| policy_entropy     | 0.7148295   |\n",
      "| policy_loss        | -0.01157851 |\n",
      "| serial_timesteps   | 340480      |\n",
      "| time_elapsed       | 1.45e+04    |\n",
      "| total_timesteps    | 340480      |\n",
      "| value_loss         | 0.44998333  |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------\n",
      "| approxkl           | 0.057209115 |\n",
      "| clipfrac           | 0.47070312  |\n",
      "| explained_variance | 0.891       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 2670        |\n",
      "| policy_entropy     | 0.95675594  |\n",
      "| policy_loss        | 0.016460638 |\n",
      "| serial_timesteps   | 341760      |\n",
      "| time_elapsed       | 1.45e+04    |\n",
      "| total_timesteps    | 341760      |\n",
      "| value_loss         | 0.34463543  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.026141992 |\n",
      "| clipfrac           | 0.19921875  |\n",
      "| explained_variance | 0.698       |\n",
      "| fps                | 28          |\n",
      "| n_updates          | 2680        |\n",
      "| policy_entropy     | 0.566998    |\n",
      "| policy_loss        | 0.017871862 |\n",
      "| serial_timesteps   | 343040      |\n",
      "| time_elapsed       | 1.46e+04    |\n",
      "| total_timesteps    | 343040      |\n",
      "| value_loss         | 1.2191014   |\n",
      "------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.008325599    |\n",
      "| clipfrac           | 0.099609375    |\n",
      "| explained_variance | 0.986          |\n",
      "| fps                | 22             |\n",
      "| n_updates          | 2690           |\n",
      "| policy_entropy     | 0.68281484     |\n",
      "| policy_loss        | -0.00068474375 |\n",
      "| serial_timesteps   | 344320         |\n",
      "| time_elapsed       | 1.46e+04       |\n",
      "| total_timesteps    | 344320         |\n",
      "| value_loss         | 7.4577417      |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.045910332    |\n",
      "| clipfrac           | 0.29882812     |\n",
      "| explained_variance | 0.967          |\n",
      "| fps                | 30             |\n",
      "| n_updates          | 2700           |\n",
      "| policy_entropy     | 0.797314       |\n",
      "| policy_loss        | -0.00088190055 |\n",
      "| serial_timesteps   | 345600         |\n",
      "| time_elapsed       | 1.47e+04       |\n",
      "| total_timesteps    | 345600         |\n",
      "| value_loss         | 0.27404112     |\n",
      "---------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.023412425   |\n",
      "| clipfrac           | 0.16601562    |\n",
      "| explained_variance | 0.42          |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 2710          |\n",
      "| policy_entropy     | 1.0830256     |\n",
      "| policy_loss        | -0.0023700562 |\n",
      "| serial_timesteps   | 346880        |\n",
      "| time_elapsed       | 1.47e+04      |\n",
      "| total_timesteps    | 346880        |\n",
      "| value_loss         | 2.3367229     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.014849096   |\n",
      "| clipfrac           | 0.22070312    |\n",
      "| explained_variance | 0.974         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 2720          |\n",
      "| policy_entropy     | 0.70853734    |\n",
      "| policy_loss        | -0.0030537022 |\n",
      "| serial_timesteps   | 348160        |\n",
      "| time_elapsed       | 1.48e+04      |\n",
      "| total_timesteps    | 348160        |\n",
      "| value_loss         | 3.127224      |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0061693005  |\n",
      "| clipfrac           | 0.09375       |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 2730          |\n",
      "| policy_entropy     | 0.7259749     |\n",
      "| policy_loss        | -0.0014146557 |\n",
      "| serial_timesteps   | 349440        |\n",
      "| time_elapsed       | 1.48e+04      |\n",
      "| total_timesteps    | 349440        |\n",
      "| value_loss         | 0.58603644    |\n",
      "--------------------------------------\n",
      "Eval num_timesteps=350000, episode_reward=551.50 +/- 1.34\n",
      "Episode length: 592.40 +/- 1.50\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0054016765 |\n",
      "| clipfrac           | 0.0546875    |\n",
      "| explained_variance | 0.95         |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 2740         |\n",
      "| policy_entropy     | 0.91384685   |\n",
      "| policy_loss        | 0.0010187298 |\n",
      "| serial_timesteps   | 350720       |\n",
      "| time_elapsed       | 1.5e+04      |\n",
      "| total_timesteps    | 350720       |\n",
      "| value_loss         | 0.43250227   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.009690041 |\n",
      "| clipfrac           | 0.12890625  |\n",
      "| explained_variance | 0.996       |\n",
      "| fps                | 24          |\n",
      "| n_updates          | 2750        |\n",
      "| policy_entropy     | 1.0359597   |\n",
      "| policy_loss        | 0.006623385 |\n",
      "| serial_timesteps   | 352000      |\n",
      "| time_elapsed       | 1.5e+04     |\n",
      "| total_timesteps    | 352000      |\n",
      "| value_loss         | 0.18625593  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.03219591  |\n",
      "| clipfrac           | 0.36523438  |\n",
      "| explained_variance | 0.747       |\n",
      "| fps                | 33          |\n",
      "| n_updates          | 2760        |\n",
      "| policy_entropy     | 1.0669727   |\n",
      "| policy_loss        | 0.012735075 |\n",
      "| serial_timesteps   | 353280      |\n",
      "| time_elapsed       | 1.5e+04     |\n",
      "| total_timesteps    | 353280      |\n",
      "| value_loss         | 5.4753156   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.011283795   |\n",
      "| clipfrac           | 0.14257812    |\n",
      "| explained_variance | 0.937         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 2770          |\n",
      "| policy_entropy     | 1.0179268     |\n",
      "| policy_loss        | -0.0025738007 |\n",
      "| serial_timesteps   | 354560        |\n",
      "| time_elapsed       | 1.51e+04      |\n",
      "| total_timesteps    | 354560        |\n",
      "| value_loss         | 0.68789816    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0077160066 |\n",
      "| clipfrac           | 0.080078125  |\n",
      "| explained_variance | 0.972        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 2780         |\n",
      "| policy_entropy     | 0.49552494   |\n",
      "| policy_loss        | -0.008200379 |\n",
      "| serial_timesteps   | 355840       |\n",
      "| time_elapsed       | 1.51e+04     |\n",
      "| total_timesteps    | 355840       |\n",
      "| value_loss         | 2.1435614    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.034788363  |\n",
      "| clipfrac           | 0.33203125   |\n",
      "| explained_variance | 0.991        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 2790         |\n",
      "| policy_entropy     | 0.7426819    |\n",
      "| policy_loss        | 0.0002983485 |\n",
      "| serial_timesteps   | 357120       |\n",
      "| time_elapsed       | 1.52e+04     |\n",
      "| total_timesteps    | 357120       |\n",
      "| value_loss         | 1.0251237    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.004620078  |\n",
      "| clipfrac           | 0.06640625   |\n",
      "| explained_variance | 0.737        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 2800         |\n",
      "| policy_entropy     | 0.70306826   |\n",
      "| policy_loss        | 0.0013221279 |\n",
      "| serial_timesteps   | 358400       |\n",
      "| time_elapsed       | 1.52e+04     |\n",
      "| total_timesteps    | 358400       |\n",
      "| value_loss         | 0.91025376   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.04703843    |\n",
      "| clipfrac           | 0.24023438    |\n",
      "| explained_variance | 0.91          |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 2810          |\n",
      "| policy_entropy     | 0.6377687     |\n",
      "| policy_loss        | -0.0039316984 |\n",
      "| serial_timesteps   | 359680        |\n",
      "| time_elapsed       | 1.53e+04      |\n",
      "| total_timesteps    | 359680        |\n",
      "| value_loss         | 6.2042065     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.017436842  |\n",
      "| clipfrac           | 0.1953125    |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 2820         |\n",
      "| policy_entropy     | 0.64010817   |\n",
      "| policy_loss        | -0.028254636 |\n",
      "| serial_timesteps   | 360960       |\n",
      "| time_elapsed       | 1.53e+04     |\n",
      "| total_timesteps    | 360960       |\n",
      "| value_loss         | 2.0871005    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.009600656  |\n",
      "| clipfrac           | 0.11328125   |\n",
      "| explained_variance | 0.765        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 2830         |\n",
      "| policy_entropy     | 0.6867082    |\n",
      "| policy_loss        | -0.010246614 |\n",
      "| serial_timesteps   | 362240       |\n",
      "| time_elapsed       | 1.54e+04     |\n",
      "| total_timesteps    | 362240       |\n",
      "| value_loss         | 3.663067     |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------\n",
      "| approxkl           | 0.010947537   |\n",
      "| clipfrac           | 0.140625      |\n",
      "| explained_variance | 0.903         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 2840          |\n",
      "| policy_entropy     | 0.8441814     |\n",
      "| policy_loss        | -0.0032981706 |\n",
      "| serial_timesteps   | 363520        |\n",
      "| time_elapsed       | 1.54e+04      |\n",
      "| total_timesteps    | 363520        |\n",
      "| value_loss         | 5.2420874     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.058181133   |\n",
      "| clipfrac           | 0.27734375    |\n",
      "| explained_variance | 0.874         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 2850          |\n",
      "| policy_entropy     | 0.68286985    |\n",
      "| policy_loss        | -0.0055250702 |\n",
      "| serial_timesteps   | 364800        |\n",
      "| time_elapsed       | 1.55e+04      |\n",
      "| total_timesteps    | 364800        |\n",
      "| value_loss         | 0.81731844    |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.02338783  |\n",
      "| clipfrac           | 0.22070312  |\n",
      "| explained_variance | 0.273       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 2860        |\n",
      "| policy_entropy     | 1.0432918   |\n",
      "| policy_loss        | 0.006716463 |\n",
      "| serial_timesteps   | 366080      |\n",
      "| time_elapsed       | 1.55e+04    |\n",
      "| total_timesteps    | 366080      |\n",
      "| value_loss         | 2.051322    |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.011216484   |\n",
      "| clipfrac           | 0.13671875    |\n",
      "| explained_variance | 0.933         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 2870          |\n",
      "| policy_entropy     | 1.046299      |\n",
      "| policy_loss        | -0.0058383835 |\n",
      "| serial_timesteps   | 367360        |\n",
      "| time_elapsed       | 1.56e+04      |\n",
      "| total_timesteps    | 367360        |\n",
      "| value_loss         | 1.5094044     |\n",
      "--------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.0061963624   |\n",
      "| clipfrac           | 0.07421875     |\n",
      "| explained_variance | 0.928          |\n",
      "| fps                | 30             |\n",
      "| n_updates          | 2880           |\n",
      "| policy_entropy     | 0.9707933      |\n",
      "| policy_loss        | -0.00066433323 |\n",
      "| serial_timesteps   | 368640         |\n",
      "| time_elapsed       | 1.56e+04       |\n",
      "| total_timesteps    | 368640         |\n",
      "| value_loss         | 1.5185547      |\n",
      "---------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.028970381  |\n",
      "| clipfrac           | 0.28125      |\n",
      "| explained_variance | -2.17        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 2890         |\n",
      "| policy_entropy     | 0.7332713    |\n",
      "| policy_loss        | -0.010473744 |\n",
      "| serial_timesteps   | 369920       |\n",
      "| time_elapsed       | 1.57e+04     |\n",
      "| total_timesteps    | 369920       |\n",
      "| value_loss         | 5.3586717    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.00930106  |\n",
      "| clipfrac           | 0.095703125 |\n",
      "| explained_variance | 0.984       |\n",
      "| fps                | 22          |\n",
      "| n_updates          | 2900        |\n",
      "| policy_entropy     | 0.36571717  |\n",
      "| policy_loss        | -0.00605293 |\n",
      "| serial_timesteps   | 371200      |\n",
      "| time_elapsed       | 1.57e+04    |\n",
      "| total_timesteps    | 371200      |\n",
      "| value_loss         | 1.9956666   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.045262653 |\n",
      "| clipfrac           | 0.29492188  |\n",
      "| explained_variance | 0.986       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 2910        |\n",
      "| policy_entropy     | 0.72173524  |\n",
      "| policy_loss        | 0.018068518 |\n",
      "| serial_timesteps   | 372480      |\n",
      "| time_elapsed       | 1.58e+04    |\n",
      "| total_timesteps    | 372480      |\n",
      "| value_loss         | 0.44765174  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.012672152  |\n",
      "| clipfrac           | 0.18359375   |\n",
      "| explained_variance | 0.794        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 2920         |\n",
      "| policy_entropy     | 0.53375185   |\n",
      "| policy_loss        | -0.024129037 |\n",
      "| serial_timesteps   | 373760       |\n",
      "| time_elapsed       | 1.58e+04     |\n",
      "| total_timesteps    | 373760       |\n",
      "| value_loss         | 127.65022    |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=375000, episode_reward=326.08 +/- 8.27\n",
      "Episode length: 502.00 +/- 3.95\n",
      "--------------------------------------\n",
      "| approxkl           | 0.015075673   |\n",
      "| clipfrac           | 0.23632812    |\n",
      "| explained_variance | 0.723         |\n",
      "| fps                | 1             |\n",
      "| n_updates          | 2930          |\n",
      "| policy_entropy     | 0.9194701     |\n",
      "| policy_loss        | 0.00017721578 |\n",
      "| serial_timesteps   | 375040        |\n",
      "| time_elapsed       | 1.58e+04      |\n",
      "| total_timesteps    | 375040        |\n",
      "| value_loss         | 4.7052855     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.036223806 |\n",
      "| clipfrac           | 0.328125    |\n",
      "| explained_variance | 0.961       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 2940        |\n",
      "| policy_entropy     | 0.740471    |\n",
      "| policy_loss        | 0.018949587 |\n",
      "| serial_timesteps   | 376320      |\n",
      "| time_elapsed       | 1.6e+04     |\n",
      "| total_timesteps    | 376320      |\n",
      "| value_loss         | 3.1225955   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.012400529   |\n",
      "| clipfrac           | 0.203125      |\n",
      "| explained_variance | 0.94          |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 2950          |\n",
      "| policy_entropy     | 0.99200004    |\n",
      "| policy_loss        | 5.4465374e-05 |\n",
      "| serial_timesteps   | 377600        |\n",
      "| time_elapsed       | 1.6e+04       |\n",
      "| total_timesteps    | 377600        |\n",
      "| value_loss         | 2.1052651     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.020135518  |\n",
      "| clipfrac           | 0.13085938   |\n",
      "| explained_variance | 0.921        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 2960         |\n",
      "| policy_entropy     | 0.44968626   |\n",
      "| policy_loss        | 0.0050306357 |\n",
      "| serial_timesteps   | 378880       |\n",
      "| time_elapsed       | 1.61e+04     |\n",
      "| total_timesteps    | 378880       |\n",
      "| value_loss         | 2.5895247    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.05069333   |\n",
      "| clipfrac           | 0.5058594    |\n",
      "| explained_variance | 0.966        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 2970         |\n",
      "| policy_entropy     | 0.51957035   |\n",
      "| policy_loss        | -0.010438468 |\n",
      "| serial_timesteps   | 380160       |\n",
      "| time_elapsed       | 1.61e+04     |\n",
      "| total_timesteps    | 380160       |\n",
      "| value_loss         | 0.6161863    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0051830774 |\n",
      "| clipfrac           | 0.064453125  |\n",
      "| explained_variance | 0.553        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 2980         |\n",
      "| policy_entropy     | 0.50114006   |\n",
      "| policy_loss        | 0.0019380957 |\n",
      "| serial_timesteps   | 381440       |\n",
      "| time_elapsed       | 1.62e+04     |\n",
      "| total_timesteps    | 381440       |\n",
      "| value_loss         | 3.2624962    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.025808401  |\n",
      "| clipfrac           | 0.33398438   |\n",
      "| explained_variance | 0.992        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 2990         |\n",
      "| policy_entropy     | 0.92780125   |\n",
      "| policy_loss        | -0.019938929 |\n",
      "| serial_timesteps   | 382720       |\n",
      "| time_elapsed       | 1.62e+04     |\n",
      "| total_timesteps    | 382720       |\n",
      "| value_loss         | 2.6181805    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.01614655  |\n",
      "| clipfrac           | 0.2109375   |\n",
      "| explained_variance | 0.836       |\n",
      "| fps                | 31          |\n",
      "| n_updates          | 3000        |\n",
      "| policy_entropy     | 0.55969083  |\n",
      "| policy_loss        | 0.004477212 |\n",
      "| serial_timesteps   | 384000      |\n",
      "| time_elapsed       | 1.63e+04    |\n",
      "| total_timesteps    | 384000      |\n",
      "| value_loss         | 2.1259966   |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.0101566315 |\n",
      "| clipfrac           | 0.18164062   |\n",
      "| explained_variance | 0.917        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 3010         |\n",
      "| policy_entropy     | 0.8500547    |\n",
      "| policy_loss        | 0.005444507  |\n",
      "| serial_timesteps   | 385280       |\n",
      "| time_elapsed       | 1.63e+04     |\n",
      "| total_timesteps    | 385280       |\n",
      "| value_loss         | 3.6963053    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0057901936 |\n",
      "| clipfrac           | 0.103515625  |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 3020         |\n",
      "| policy_entropy     | 0.9901601    |\n",
      "| policy_loss        | -0.016367286 |\n",
      "| serial_timesteps   | 386560       |\n",
      "| time_elapsed       | 1.64e+04     |\n",
      "| total_timesteps    | 386560       |\n",
      "| value_loss         | 0.9711213    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.037545368 |\n",
      "| clipfrac           | 0.35351562  |\n",
      "| explained_variance | 0.923       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 3030        |\n",
      "| policy_entropy     | 0.8679892   |\n",
      "| policy_loss        | 0.013235854 |\n",
      "| serial_timesteps   | 387840      |\n",
      "| time_elapsed       | 1.64e+04    |\n",
      "| total_timesteps    | 387840      |\n",
      "| value_loss         | 0.21670452  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.004033137  |\n",
      "| clipfrac           | 0.064453125  |\n",
      "| explained_variance | 0.985        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 3040         |\n",
      "| policy_entropy     | 0.64479345   |\n",
      "| policy_loss        | -0.006522122 |\n",
      "| serial_timesteps   | 389120       |\n",
      "| time_elapsed       | 1.64e+04     |\n",
      "| total_timesteps    | 389120       |\n",
      "| value_loss         | 0.34420925   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0030786     |\n",
      "| clipfrac           | 0.037109375   |\n",
      "| explained_variance | 0.991         |\n",
      "| fps                | 21            |\n",
      "| n_updates          | 3050          |\n",
      "| policy_entropy     | 0.65829307    |\n",
      "| policy_loss        | -0.0043323305 |\n",
      "| serial_timesteps   | 390400        |\n",
      "| time_elapsed       | 1.65e+04      |\n",
      "| total_timesteps    | 390400        |\n",
      "| value_loss         | 2.1346211     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.02640002    |\n",
      "| clipfrac           | 0.32617188    |\n",
      "| explained_variance | 0.99          |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 3060          |\n",
      "| policy_entropy     | 0.90751225    |\n",
      "| policy_loss        | 0.00013301591 |\n",
      "| serial_timesteps   | 391680        |\n",
      "| time_elapsed       | 1.65e+04      |\n",
      "| total_timesteps    | 391680        |\n",
      "| value_loss         | 0.63291       |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.014189217  |\n",
      "| clipfrac           | 0.13867188   |\n",
      "| explained_variance | 0.894        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 3070         |\n",
      "| policy_entropy     | 0.64619267   |\n",
      "| policy_loss        | -0.014878202 |\n",
      "| serial_timesteps   | 392960       |\n",
      "| time_elapsed       | 1.66e+04     |\n",
      "| total_timesteps    | 392960       |\n",
      "| value_loss         | 0.73027176   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.04519734   |\n",
      "| clipfrac           | 0.30664062   |\n",
      "| explained_variance | 0.995        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 3080         |\n",
      "| policy_entropy     | 1.0065694    |\n",
      "| policy_loss        | -0.014400279 |\n",
      "| serial_timesteps   | 394240       |\n",
      "| time_elapsed       | 1.66e+04     |\n",
      "| total_timesteps    | 394240       |\n",
      "| value_loss         | 1.0182976    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.022781225  |\n",
      "| clipfrac           | 0.22265625   |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 3090         |\n",
      "| policy_entropy     | 1.0350177    |\n",
      "| policy_loss        | -0.003860369 |\n",
      "| serial_timesteps   | 395520       |\n",
      "| time_elapsed       | 1.67e+04     |\n",
      "| total_timesteps    | 395520       |\n",
      "| value_loss         | 0.34393468   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.03673352  |\n",
      "| clipfrac           | 0.33398438  |\n",
      "| explained_variance | 0.85        |\n",
      "| fps                | 31          |\n",
      "| n_updates          | 3100        |\n",
      "| policy_entropy     | 1.0123163   |\n",
      "| policy_loss        | 0.011826647 |\n",
      "| serial_timesteps   | 396800      |\n",
      "| time_elapsed       | 1.67e+04    |\n",
      "| total_timesteps    | 396800      |\n",
      "| value_loss         | 0.26720977  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.039119035  |\n",
      "| clipfrac           | 0.31640625   |\n",
      "| explained_variance | 0.979        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 3110         |\n",
      "| policy_entropy     | 0.9843302    |\n",
      "| policy_loss        | -0.015885865 |\n",
      "| serial_timesteps   | 398080       |\n",
      "| time_elapsed       | 1.68e+04     |\n",
      "| total_timesteps    | 398080       |\n",
      "| value_loss         | 3.7932687    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.036587894  |\n",
      "| clipfrac           | 0.32617188   |\n",
      "| explained_variance | 0.984        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 3120         |\n",
      "| policy_entropy     | 0.81009483   |\n",
      "| policy_loss        | 0.0016460605 |\n",
      "| serial_timesteps   | 399360       |\n",
      "| time_elapsed       | 1.68e+04     |\n",
      "| total_timesteps    | 399360       |\n",
      "| value_loss         | 0.2814755    |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=400000, episode_reward=449.40 +/- 5.52\n",
      "Episode length: 583.00 +/- 1.41\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0175765    |\n",
      "| clipfrac           | 0.23242188   |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 3130         |\n",
      "| policy_entropy     | 0.7374327    |\n",
      "| policy_loss        | -0.015775604 |\n",
      "| serial_timesteps   | 400640       |\n",
      "| time_elapsed       | 1.7e+04      |\n",
      "| total_timesteps    | 400640       |\n",
      "| value_loss         | 1.7281259    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.008175505   |\n",
      "| clipfrac           | 0.0859375     |\n",
      "| explained_variance | 0.737         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 3140          |\n",
      "| policy_entropy     | 0.39381334    |\n",
      "| policy_loss        | -0.0013978756 |\n",
      "| serial_timesteps   | 401920        |\n",
      "| time_elapsed       | 1.7e+04       |\n",
      "| total_timesteps    | 401920        |\n",
      "| value_loss         | 138.93086     |\n",
      "--------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.0011839095   |\n",
      "| clipfrac           | 0.005859375    |\n",
      "| explained_variance | 0.976          |\n",
      "| fps                | 29             |\n",
      "| n_updates          | 3150           |\n",
      "| policy_entropy     | 0.46693414     |\n",
      "| policy_loss        | -0.00031947636 |\n",
      "| serial_timesteps   | 403200         |\n",
      "| time_elapsed       | 1.71e+04       |\n",
      "| total_timesteps    | 403200         |\n",
      "| value_loss         | 0.55299395     |\n",
      "---------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.011378551 |\n",
      "| clipfrac           | 0.20898438  |\n",
      "| explained_variance | 0.804       |\n",
      "| fps                | 22          |\n",
      "| n_updates          | 3160        |\n",
      "| policy_entropy     | 0.79757416  |\n",
      "| policy_loss        | -0.01568296 |\n",
      "| serial_timesteps   | 404480      |\n",
      "| time_elapsed       | 1.71e+04    |\n",
      "| total_timesteps    | 404480      |\n",
      "| value_loss         | 1.7145345   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.016745547 |\n",
      "| clipfrac           | 0.21875     |\n",
      "| explained_variance | 0.809       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 3170        |\n",
      "| policy_entropy     | 0.72666335  |\n",
      "| policy_loss        | 0.005309361 |\n",
      "| serial_timesteps   | 405760      |\n",
      "| time_elapsed       | 1.72e+04    |\n",
      "| total_timesteps    | 405760      |\n",
      "| value_loss         | 3.4195704   |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.020802116  |\n",
      "| clipfrac           | 0.21875      |\n",
      "| explained_variance | 0.384        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 3180         |\n",
      "| policy_entropy     | 0.5253575    |\n",
      "| policy_loss        | -0.005836417 |\n",
      "| serial_timesteps   | 407040       |\n",
      "| time_elapsed       | 1.72e+04     |\n",
      "| total_timesteps    | 407040       |\n",
      "| value_loss         | 2.757131     |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0034693694  |\n",
      "| clipfrac           | 0.03515625    |\n",
      "| explained_variance | 0.988         |\n",
      "| fps                | 21            |\n",
      "| n_updates          | 3190          |\n",
      "| policy_entropy     | 0.7588762     |\n",
      "| policy_loss        | -0.0067353025 |\n",
      "| serial_timesteps   | 408320        |\n",
      "| time_elapsed       | 1.72e+04      |\n",
      "| total_timesteps    | 408320        |\n",
      "| value_loss         | 2.4023001     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.013527912   |\n",
      "| clipfrac           | 0.22851562    |\n",
      "| explained_variance | 0.903         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 3200          |\n",
      "| policy_entropy     | 1.0144371     |\n",
      "| policy_loss        | 1.1369353e-05 |\n",
      "| serial_timesteps   | 409600        |\n",
      "| time_elapsed       | 1.73e+04      |\n",
      "| total_timesteps    | 409600        |\n",
      "| value_loss         | 0.34255186    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.013725749   |\n",
      "| clipfrac           | 0.17773438    |\n",
      "| explained_variance | 0.954         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 3210          |\n",
      "| policy_entropy     | 0.912458      |\n",
      "| policy_loss        | -0.0067314114 |\n",
      "| serial_timesteps   | 410880        |\n",
      "| time_elapsed       | 1.73e+04      |\n",
      "| total_timesteps    | 410880        |\n",
      "| value_loss         | 1.0520453     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0062948545 |\n",
      "| clipfrac           | 0.064453125  |\n",
      "| explained_variance | 0.946        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 3220         |\n",
      "| policy_entropy     | 0.8896655    |\n",
      "| policy_loss        | -0.005798567 |\n",
      "| serial_timesteps   | 412160       |\n",
      "| time_elapsed       | 1.74e+04     |\n",
      "| total_timesteps    | 412160       |\n",
      "| value_loss         | 1.5769377    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.018821655 |\n",
      "| clipfrac           | 0.27929688  |\n",
      "| explained_variance | 0.769       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 3230        |\n",
      "| policy_entropy     | 0.88853145  |\n",
      "| policy_loss        | 0.004701058 |\n",
      "| serial_timesteps   | 413440      |\n",
      "| time_elapsed       | 1.74e+04    |\n",
      "| total_timesteps    | 413440      |\n",
      "| value_loss         | 0.24908432  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.016608     |\n",
      "| clipfrac           | 0.20898438   |\n",
      "| explained_variance | 0.977        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 3240         |\n",
      "| policy_entropy     | 0.90193313   |\n",
      "| policy_loss        | -0.004788656 |\n",
      "| serial_timesteps   | 414720       |\n",
      "| time_elapsed       | 1.75e+04     |\n",
      "| total_timesteps    | 414720       |\n",
      "| value_loss         | 0.599787     |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0055063874  |\n",
      "| clipfrac           | 0.078125      |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 3250          |\n",
      "| policy_entropy     | 0.8381557     |\n",
      "| policy_loss        | -0.0040528392 |\n",
      "| serial_timesteps   | 416000        |\n",
      "| time_elapsed       | 1.75e+04      |\n",
      "| total_timesteps    | 416000        |\n",
      "| value_loss         | 0.68435216    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.01272923   |\n",
      "| clipfrac           | 0.16210938   |\n",
      "| explained_variance | 0.936        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 3260         |\n",
      "| policy_entropy     | 0.6820276    |\n",
      "| policy_loss        | 0.0034005092 |\n",
      "| serial_timesteps   | 417280       |\n",
      "| time_elapsed       | 1.76e+04     |\n",
      "| total_timesteps    | 417280       |\n",
      "| value_loss         | 0.3424793    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.012671278  |\n",
      "| clipfrac           | 0.19726562   |\n",
      "| explained_variance | 0.974        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 3270         |\n",
      "| policy_entropy     | 0.8967742    |\n",
      "| policy_loss        | -0.009403996 |\n",
      "| serial_timesteps   | 418560       |\n",
      "| time_elapsed       | 1.76e+04     |\n",
      "| total_timesteps    | 418560       |\n",
      "| value_loss         | 2.4740171    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.010130165  |\n",
      "| clipfrac           | 0.1328125    |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 3280         |\n",
      "| policy_entropy     | 0.48205635   |\n",
      "| policy_loss        | -0.016661432 |\n",
      "| serial_timesteps   | 419840       |\n",
      "| time_elapsed       | 1.77e+04     |\n",
      "| total_timesteps    | 419840       |\n",
      "| value_loss         | 2.4824793    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.01752058   |\n",
      "| clipfrac           | 0.24023438   |\n",
      "| explained_variance | 0.992        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 3290         |\n",
      "| policy_entropy     | 0.9279757    |\n",
      "| policy_loss        | -0.015285248 |\n",
      "| serial_timesteps   | 421120       |\n",
      "| time_elapsed       | 1.77e+04     |\n",
      "| total_timesteps    | 421120       |\n",
      "| value_loss         | 0.48854345   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0246118    |\n",
      "| clipfrac           | 0.19140625   |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 3300         |\n",
      "| policy_entropy     | 0.48210046   |\n",
      "| policy_loss        | -0.017893745 |\n",
      "| serial_timesteps   | 422400       |\n",
      "| time_elapsed       | 1.78e+04     |\n",
      "| total_timesteps    | 422400       |\n",
      "| value_loss         | 1.541881     |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.035158508 |\n",
      "| clipfrac           | 0.15039062  |\n",
      "| explained_variance | 0.313       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 3310        |\n",
      "| policy_entropy     | 0.86941546  |\n",
      "| policy_loss        | 0.022033278 |\n",
      "| serial_timesteps   | 423680      |\n",
      "| time_elapsed       | 1.78e+04    |\n",
      "| total_timesteps    | 423680      |\n",
      "| value_loss         | 0.50422245  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.012091196  |\n",
      "| clipfrac           | 0.1484375    |\n",
      "| explained_variance | 0.968        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 3320         |\n",
      "| policy_entropy     | 0.6352074    |\n",
      "| policy_loss        | -0.004068819 |\n",
      "| serial_timesteps   | 424960       |\n",
      "| time_elapsed       | 1.79e+04     |\n",
      "| total_timesteps    | 424960       |\n",
      "| value_loss         | 0.5481961    |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=425000, episode_reward=532.10 +/- 0.73\n",
      "Episode length: 592.00 +/- 0.63\n",
      "--------------------------------------\n",
      "| approxkl           | 0.017740093   |\n",
      "| clipfrac           | 0.17773438    |\n",
      "| explained_variance | 0.981         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 3330          |\n",
      "| policy_entropy     | 0.63262033    |\n",
      "| policy_loss        | 0.00019104406 |\n",
      "| serial_timesteps   | 426240        |\n",
      "| time_elapsed       | 1.8e+04       |\n",
      "| total_timesteps    | 426240        |\n",
      "| value_loss         | 0.8558919     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.037332162 |\n",
      "| clipfrac           | 0.23828125  |\n",
      "| explained_variance | 0.872       |\n",
      "| fps                | 31          |\n",
      "| n_updates          | 3340        |\n",
      "| policy_entropy     | 0.41731808  |\n",
      "| policy_loss        | 0.025223594 |\n",
      "| serial_timesteps   | 427520      |\n",
      "| time_elapsed       | 1.81e+04    |\n",
      "| total_timesteps    | 427520      |\n",
      "| value_loss         | 0.65551925  |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.0044229175 |\n",
      "| clipfrac           | 0.068359375  |\n",
      "| explained_variance | 0.98         |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 3350         |\n",
      "| policy_entropy     | 0.3496924    |\n",
      "| policy_loss        | -0.011557536 |\n",
      "| serial_timesteps   | 428800       |\n",
      "| time_elapsed       | 1.81e+04     |\n",
      "| total_timesteps    | 428800       |\n",
      "| value_loss         | 1.1255476    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0049738702  |\n",
      "| clipfrac           | 0.068359375   |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 3360          |\n",
      "| policy_entropy     | 0.47203577    |\n",
      "| policy_loss        | -0.0072365976 |\n",
      "| serial_timesteps   | 430080        |\n",
      "| time_elapsed       | 1.82e+04      |\n",
      "| total_timesteps    | 430080        |\n",
      "| value_loss         | 0.6553848     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0076737963 |\n",
      "| clipfrac           | 0.10546875   |\n",
      "| explained_variance | 0.824        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 3370         |\n",
      "| policy_entropy     | 0.4601811    |\n",
      "| policy_loss        | -0.002294349 |\n",
      "| serial_timesteps   | 431360       |\n",
      "| time_elapsed       | 1.82e+04     |\n",
      "| total_timesteps    | 431360       |\n",
      "| value_loss         | 0.8360673    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.03239326  |\n",
      "| clipfrac           | 0.2421875   |\n",
      "| explained_variance | 0.984       |\n",
      "| fps                | 22          |\n",
      "| n_updates          | 3380        |\n",
      "| policy_entropy     | 0.752596    |\n",
      "| policy_loss        | -0.00980786 |\n",
      "| serial_timesteps   | 432640      |\n",
      "| time_elapsed       | 1.82e+04    |\n",
      "| total_timesteps    | 432640      |\n",
      "| value_loss         | 2.2908692   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.020080775 |\n",
      "| clipfrac           | 0.203125    |\n",
      "| explained_variance | 0.879       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 3390        |\n",
      "| policy_entropy     | 0.6739564   |\n",
      "| policy_loss        | 0.021668604 |\n",
      "| serial_timesteps   | 433920      |\n",
      "| time_elapsed       | 1.83e+04    |\n",
      "| total_timesteps    | 433920      |\n",
      "| value_loss         | 0.8250059   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.039389234 |\n",
      "| clipfrac           | 0.35546875  |\n",
      "| explained_variance | 0.796       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 3400        |\n",
      "| policy_entropy     | 0.51547253  |\n",
      "| policy_loss        | 0.021308718 |\n",
      "| serial_timesteps   | 435200      |\n",
      "| time_elapsed       | 1.83e+04    |\n",
      "| total_timesteps    | 435200      |\n",
      "| value_loss         | 2.7201848   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.019427527  |\n",
      "| clipfrac           | 0.203125     |\n",
      "| explained_variance | 0.869        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 3410         |\n",
      "| policy_entropy     | 0.47603607   |\n",
      "| policy_loss        | -0.010296372 |\n",
      "| serial_timesteps   | 436480       |\n",
      "| time_elapsed       | 1.84e+04     |\n",
      "| total_timesteps    | 436480       |\n",
      "| value_loss         | 99.39715     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0144618945 |\n",
      "| clipfrac           | 0.09765625   |\n",
      "| explained_variance | 0.77         |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 3420         |\n",
      "| policy_entropy     | 0.66520214   |\n",
      "| policy_loss        | -0.005892937 |\n",
      "| serial_timesteps   | 437760       |\n",
      "| time_elapsed       | 1.84e+04     |\n",
      "| total_timesteps    | 437760       |\n",
      "| value_loss         | 6.912055     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.002295205  |\n",
      "| clipfrac           | 0.04296875   |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 3430         |\n",
      "| policy_entropy     | 0.4081654    |\n",
      "| policy_loss        | -0.006003817 |\n",
      "| serial_timesteps   | 439040       |\n",
      "| time_elapsed       | 1.85e+04     |\n",
      "| total_timesteps    | 439040       |\n",
      "| value_loss         | 0.6373812    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.016846381 |\n",
      "| clipfrac           | 0.16992188  |\n",
      "| explained_variance | -0.696      |\n",
      "| fps                | 31          |\n",
      "| n_updates          | 3440        |\n",
      "| policy_entropy     | 0.73664844  |\n",
      "| policy_loss        | 0.008472658 |\n",
      "| serial_timesteps   | 440320      |\n",
      "| time_elapsed       | 1.85e+04    |\n",
      "| total_timesteps    | 440320      |\n",
      "| value_loss         | 4.198907    |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.005084568 |\n",
      "| clipfrac           | 0.080078125 |\n",
      "| explained_variance | 0.95        |\n",
      "| fps                | 23          |\n",
      "| n_updates          | 3450        |\n",
      "| policy_entropy     | 0.5331526   |\n",
      "| policy_loss        | 0.003999968 |\n",
      "| serial_timesteps   | 441600      |\n",
      "| time_elapsed       | 1.86e+04    |\n",
      "| total_timesteps    | 441600      |\n",
      "| value_loss         | 0.7480705   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0038807248 |\n",
      "| clipfrac           | 0.064453125  |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 3460         |\n",
      "| policy_entropy     | 0.5674441    |\n",
      "| policy_loss        | -0.013211219 |\n",
      "| serial_timesteps   | 442880       |\n",
      "| time_elapsed       | 1.86e+04     |\n",
      "| total_timesteps    | 442880       |\n",
      "| value_loss         | 1.3972716    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0052650524 |\n",
      "| clipfrac           | 0.072265625  |\n",
      "| explained_variance | 0.491        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 3470         |\n",
      "| policy_entropy     | 0.48983085   |\n",
      "| policy_loss        | -0.012127935 |\n",
      "| serial_timesteps   | 444160       |\n",
      "| time_elapsed       | 1.87e+04     |\n",
      "| total_timesteps    | 444160       |\n",
      "| value_loss         | 5.502101     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.005054907  |\n",
      "| clipfrac           | 0.072265625  |\n",
      "| explained_variance | 0.945        |\n",
      "| fps                | 25           |\n",
      "| n_updates          | 3480         |\n",
      "| policy_entropy     | 0.45470393   |\n",
      "| policy_loss        | 0.0013664444 |\n",
      "| serial_timesteps   | 445440       |\n",
      "| time_elapsed       | 1.87e+04     |\n",
      "| total_timesteps    | 445440       |\n",
      "| value_loss         | 4.0272       |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.06558387  |\n",
      "| clipfrac           | 0.421875    |\n",
      "| explained_variance | 0.999       |\n",
      "| fps                | 25          |\n",
      "| n_updates          | 3490        |\n",
      "| policy_entropy     | 1.0844802   |\n",
      "| policy_loss        | -0.02369388 |\n",
      "| serial_timesteps   | 446720      |\n",
      "| time_elapsed       | 1.88e+04    |\n",
      "| total_timesteps    | 446720      |\n",
      "| value_loss         | 0.79868335  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0068997424 |\n",
      "| clipfrac           | 0.095703125  |\n",
      "| explained_variance | 0.71         |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 3500         |\n",
      "| policy_entropy     | 0.48530233   |\n",
      "| policy_loss        | 0.0076895137 |\n",
      "| serial_timesteps   | 448000       |\n",
      "| time_elapsed       | 1.88e+04     |\n",
      "| total_timesteps    | 448000       |\n",
      "| value_loss         | 0.60998917   |\n",
      "-------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.006967662    |\n",
      "| clipfrac           | 0.119140625    |\n",
      "| explained_variance | 0.188          |\n",
      "| fps                | 28             |\n",
      "| n_updates          | 3510           |\n",
      "| policy_entropy     | 0.64194894     |\n",
      "| policy_loss        | -0.00050483807 |\n",
      "| serial_timesteps   | 449280         |\n",
      "| time_elapsed       | 1.89e+04       |\n",
      "| total_timesteps    | 449280         |\n",
      "| value_loss         | 3.0248961      |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eval num_timesteps=450000, episode_reward=559.50 +/- 0.32\n",
      "Episode length: 595.40 +/- 0.80\n",
      "New best mean reward!\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0028486624 |\n",
      "| clipfrac           | 0.03515625   |\n",
      "| explained_variance | 0.993        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 3520         |\n",
      "| policy_entropy     | 0.7151317    |\n",
      "| policy_loss        | -0.010568585 |\n",
      "| serial_timesteps   | 450560       |\n",
      "| time_elapsed       | 1.9e+04      |\n",
      "| total_timesteps    | 450560       |\n",
      "| value_loss         | 1.1780895    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.043073773 |\n",
      "| clipfrac           | 0.44726562  |\n",
      "| explained_variance | 0.515       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 3530        |\n",
      "| policy_entropy     | 0.8482641   |\n",
      "| policy_loss        | 0.00211457  |\n",
      "| serial_timesteps   | 451840      |\n",
      "| time_elapsed       | 1.91e+04    |\n",
      "| total_timesteps    | 451840      |\n",
      "| value_loss         | 9.530409    |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0147113195 |\n",
      "| clipfrac           | 0.18554688   |\n",
      "| explained_variance | 0.955        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 3540         |\n",
      "| policy_entropy     | 0.47128788   |\n",
      "| policy_loss        | -0.010013003 |\n",
      "| serial_timesteps   | 453120       |\n",
      "| time_elapsed       | 1.91e+04     |\n",
      "| total_timesteps    | 453120       |\n",
      "| value_loss         | 0.5006295    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.010187844  |\n",
      "| clipfrac           | 0.14453125   |\n",
      "| explained_variance | 0.975        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 3550         |\n",
      "| policy_entropy     | 0.4222576    |\n",
      "| policy_loss        | -0.008923314 |\n",
      "| serial_timesteps   | 454400       |\n",
      "| time_elapsed       | 1.91e+04     |\n",
      "| total_timesteps    | 454400       |\n",
      "| value_loss         | 1.6420265    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.023797516 |\n",
      "| clipfrac           | 0.25390625  |\n",
      "| explained_variance | 0.996       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 3560        |\n",
      "| policy_entropy     | 0.71779674  |\n",
      "| policy_loss        | 0.006539799 |\n",
      "| serial_timesteps   | 455680      |\n",
      "| time_elapsed       | 1.92e+04    |\n",
      "| total_timesteps    | 455680      |\n",
      "| value_loss         | 0.8909006   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.014597184   |\n",
      "| clipfrac           | 0.21679688    |\n",
      "| explained_variance | 0.969         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 3570          |\n",
      "| policy_entropy     | 0.7362321     |\n",
      "| policy_loss        | 0.00016219576 |\n",
      "| serial_timesteps   | 456960        |\n",
      "| time_elapsed       | 1.92e+04      |\n",
      "| total_timesteps    | 456960        |\n",
      "| value_loss         | 0.16244294    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.013727772   |\n",
      "| clipfrac           | 0.13085938    |\n",
      "| explained_variance | 0.996         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 3580          |\n",
      "| policy_entropy     | 0.6098845     |\n",
      "| policy_loss        | -0.0007574386 |\n",
      "| serial_timesteps   | 458240        |\n",
      "| time_elapsed       | 1.93e+04      |\n",
      "| total_timesteps    | 458240        |\n",
      "| value_loss         | 0.2301127     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.008704444   |\n",
      "| clipfrac           | 0.087890625   |\n",
      "| explained_variance | 0.998         |\n",
      "| fps                | 27            |\n",
      "| n_updates          | 3590          |\n",
      "| policy_entropy     | 0.79365075    |\n",
      "| policy_loss        | -0.0052602403 |\n",
      "| serial_timesteps   | 459520        |\n",
      "| time_elapsed       | 1.93e+04      |\n",
      "| total_timesteps    | 459520        |\n",
      "| value_loss         | 2.0061662     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.036779463   |\n",
      "| clipfrac           | 0.28125       |\n",
      "| explained_variance | 0.955         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 3600          |\n",
      "| policy_entropy     | 0.54199886    |\n",
      "| policy_loss        | -0.0038029635 |\n",
      "| serial_timesteps   | 460800        |\n",
      "| time_elapsed       | 1.94e+04      |\n",
      "| total_timesteps    | 460800        |\n",
      "| value_loss         | 0.43701947    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.005413574   |\n",
      "| clipfrac           | 0.0625        |\n",
      "| explained_variance | 0.958         |\n",
      "| fps                | 26            |\n",
      "| n_updates          | 3610          |\n",
      "| policy_entropy     | 0.81629026    |\n",
      "| policy_loss        | -0.0094764875 |\n",
      "| serial_timesteps   | 462080        |\n",
      "| time_elapsed       | 1.94e+04      |\n",
      "| total_timesteps    | 462080        |\n",
      "| value_loss         | 0.8471861     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.037883833  |\n",
      "| clipfrac           | 0.2734375    |\n",
      "| explained_variance | 0.989        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 3620         |\n",
      "| policy_entropy     | 0.95560074   |\n",
      "| policy_loss        | -0.005684565 |\n",
      "| serial_timesteps   | 463360       |\n",
      "| time_elapsed       | 1.95e+04     |\n",
      "| total_timesteps    | 463360       |\n",
      "| value_loss         | 14.646633    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.017345145  |\n",
      "| clipfrac           | 0.20898438   |\n",
      "| explained_variance | 0.923        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 3630         |\n",
      "| policy_entropy     | 0.97449255   |\n",
      "| policy_loss        | -0.004534985 |\n",
      "| serial_timesteps   | 464640       |\n",
      "| time_elapsed       | 1.95e+04     |\n",
      "| total_timesteps    | 464640       |\n",
      "| value_loss         | 0.76777023   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.004426435   |\n",
      "| clipfrac           | 0.0625        |\n",
      "| explained_variance | 0.652         |\n",
      "| fps                | 27            |\n",
      "| n_updates          | 3640          |\n",
      "| policy_entropy     | 0.7732644     |\n",
      "| policy_loss        | -0.0030856214 |\n",
      "| serial_timesteps   | 465920        |\n",
      "| time_elapsed       | 1.96e+04      |\n",
      "| total_timesteps    | 465920        |\n",
      "| value_loss         | 3.8839445     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.024697158  |\n",
      "| clipfrac           | 0.1640625    |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 3650         |\n",
      "| policy_entropy     | 0.74438196   |\n",
      "| policy_loss        | -0.017402425 |\n",
      "| serial_timesteps   | 467200       |\n",
      "| time_elapsed       | 1.96e+04     |\n",
      "| total_timesteps    | 467200       |\n",
      "| value_loss         | 1.3111981    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.014451614 |\n",
      "| clipfrac           | 0.15820312  |\n",
      "| explained_variance | -0.281      |\n",
      "| fps                | 28          |\n",
      "| n_updates          | 3660        |\n",
      "| policy_entropy     | 0.65285295  |\n",
      "| policy_loss        | 0.008707982 |\n",
      "| serial_timesteps   | 468480      |\n",
      "| time_elapsed       | 1.97e+04    |\n",
      "| total_timesteps    | 468480      |\n",
      "| value_loss         | 7.9010425   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.011663111 |\n",
      "| clipfrac           | 0.16796875  |\n",
      "| explained_variance | 0.516       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 3670        |\n",
      "| policy_entropy     | 1.0259235   |\n",
      "| policy_loss        | -0.01290177 |\n",
      "| serial_timesteps   | 469760      |\n",
      "| time_elapsed       | 1.97e+04    |\n",
      "| total_timesteps    | 469760      |\n",
      "| value_loss         | 4.6107893   |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.07006819  |\n",
      "| clipfrac           | 0.33789062  |\n",
      "| explained_variance | 0.727       |\n",
      "| fps                | 28          |\n",
      "| n_updates          | 3680        |\n",
      "| policy_entropy     | 0.82340866  |\n",
      "| policy_loss        | 0.015811002 |\n",
      "| serial_timesteps   | 471040      |\n",
      "| time_elapsed       | 1.98e+04    |\n",
      "| total_timesteps    | 471040      |\n",
      "| value_loss         | 4.9331145   |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.020154154  |\n",
      "| clipfrac           | 0.21484375   |\n",
      "| explained_variance | 0.993        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 3690         |\n",
      "| policy_entropy     | 0.75034565   |\n",
      "| policy_loss        | -0.019639295 |\n",
      "| serial_timesteps   | 472320       |\n",
      "| time_elapsed       | 1.98e+04     |\n",
      "| total_timesteps    | 472320       |\n",
      "| value_loss         | 3.4416385    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.035683993 |\n",
      "| clipfrac           | 0.27734375  |\n",
      "| explained_variance | 0.931       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 3700        |\n",
      "| policy_entropy     | 0.6527383   |\n",
      "| policy_loss        | 0.016873267 |\n",
      "| serial_timesteps   | 473600      |\n",
      "| time_elapsed       | 1.99e+04    |\n",
      "| total_timesteps    | 473600      |\n",
      "| value_loss         | 0.32265282  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.002098152  |\n",
      "| clipfrac           | 0.015625     |\n",
      "| explained_variance | 0.643        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 3710         |\n",
      "| policy_entropy     | 0.6818691    |\n",
      "| policy_loss        | 0.0018022314 |\n",
      "| serial_timesteps   | 474880       |\n",
      "| time_elapsed       | 1.99e+04     |\n",
      "| total_timesteps    | 474880       |\n",
      "| value_loss         | 1.9805853    |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=475000, episode_reward=539.10 +/- 1.11\n",
      "Episode length: 583.80 +/- 0.75\n",
      "-------------------------------------\n",
      "| approxkl           | 0.018781263  |\n",
      "| clipfrac           | 0.19726562   |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 3720         |\n",
      "| policy_entropy     | 0.7460379    |\n",
      "| policy_loss        | 0.0053046453 |\n",
      "| serial_timesteps   | 476160       |\n",
      "| time_elapsed       | 2e+04        |\n",
      "| total_timesteps    | 476160       |\n",
      "| value_loss         | 0.52364165   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.004754085  |\n",
      "| clipfrac           | 0.05859375   |\n",
      "| explained_variance | 0.954        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 3730         |\n",
      "| policy_entropy     | 0.58037543   |\n",
      "| policy_loss        | 0.0006315047 |\n",
      "| serial_timesteps   | 477440       |\n",
      "| time_elapsed       | 2.01e+04     |\n",
      "| total_timesteps    | 477440       |\n",
      "| value_loss         | 0.2013232    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.015634727  |\n",
      "| clipfrac           | 0.1171875    |\n",
      "| explained_variance | 0.972        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 3740         |\n",
      "| policy_entropy     | 0.40817297   |\n",
      "| policy_loss        | -0.007557608 |\n",
      "| serial_timesteps   | 478720       |\n",
      "| time_elapsed       | 2.01e+04     |\n",
      "| total_timesteps    | 478720       |\n",
      "| value_loss         | 0.40307373   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.017688837 |\n",
      "| clipfrac           | 0.125       |\n",
      "| explained_variance | 0.998       |\n",
      "| fps                | 23          |\n",
      "| n_updates          | 3750        |\n",
      "| policy_entropy     | 0.88987756  |\n",
      "| policy_loss        | -0.02382865 |\n",
      "| serial_timesteps   | 480000      |\n",
      "| time_elapsed       | 2.02e+04    |\n",
      "| total_timesteps    | 480000      |\n",
      "| value_loss         | 0.50116223  |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.01412748    |\n",
      "| clipfrac           | 0.111328125   |\n",
      "| explained_variance | 0.791         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 3760          |\n",
      "| policy_entropy     | 0.6603587     |\n",
      "| policy_loss        | -0.0059428145 |\n",
      "| serial_timesteps   | 481280        |\n",
      "| time_elapsed       | 2.02e+04      |\n",
      "| total_timesteps    | 481280        |\n",
      "| value_loss         | 0.10908691    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.004180554  |\n",
      "| clipfrac           | 0.0703125    |\n",
      "| explained_variance | 0.986        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 3770         |\n",
      "| policy_entropy     | 0.6553262    |\n",
      "| policy_loss        | 0.0011721307 |\n",
      "| serial_timesteps   | 482560       |\n",
      "| time_elapsed       | 2.03e+04     |\n",
      "| total_timesteps    | 482560       |\n",
      "| value_loss         | 0.32998592   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0076864585 |\n",
      "| clipfrac           | 0.08984375   |\n",
      "| explained_variance | 0.968        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 3780         |\n",
      "| policy_entropy     | 0.6181561    |\n",
      "| policy_loss        | -0.004606813 |\n",
      "| serial_timesteps   | 483840       |\n",
      "| time_elapsed       | 2.03e+04     |\n",
      "| total_timesteps    | 483840       |\n",
      "| value_loss         | 6.468685     |\n",
      "-------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.0051374696   |\n",
      "| clipfrac           | 0.060546875    |\n",
      "| explained_variance | 0.991          |\n",
      "| fps                | 30             |\n",
      "| n_updates          | 3790           |\n",
      "| policy_entropy     | 0.37981644     |\n",
      "| policy_loss        | -0.00010538101 |\n",
      "| serial_timesteps   | 485120         |\n",
      "| time_elapsed       | 2.04e+04       |\n",
      "| total_timesteps    | 485120         |\n",
      "| value_loss         | 0.50512856     |\n",
      "---------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.009772189   |\n",
      "| clipfrac           | 0.115234375   |\n",
      "| explained_variance | 0.968         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 3800          |\n",
      "| policy_entropy     | 0.35676262    |\n",
      "| policy_loss        | -0.0151766585 |\n",
      "| serial_timesteps   | 486400        |\n",
      "| time_elapsed       | 2.04e+04      |\n",
      "| total_timesteps    | 486400        |\n",
      "| value_loss         | 0.23142509    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0036961976 |\n",
      "| clipfrac           | 0.0625       |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 3810         |\n",
      "| policy_entropy     | 0.63482577   |\n",
      "| policy_loss        | -0.012435967 |\n",
      "| serial_timesteps   | 487680       |\n",
      "| time_elapsed       | 2.05e+04     |\n",
      "| total_timesteps    | 487680       |\n",
      "| value_loss         | 0.4259109    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.038491633 |\n",
      "| clipfrac           | 0.24609375  |\n",
      "| explained_variance | 0.988       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 3820        |\n",
      "| policy_entropy     | 0.71882844  |\n",
      "| policy_loss        | -0.01796857 |\n",
      "| serial_timesteps   | 488960      |\n",
      "| time_elapsed       | 2.05e+04    |\n",
      "| total_timesteps    | 488960      |\n",
      "| value_loss         | 0.35854784  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.017625796 |\n",
      "| clipfrac           | 0.21484375  |\n",
      "| explained_variance | 0.938       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 3830        |\n",
      "| policy_entropy     | 0.90430105  |\n",
      "| policy_loss        | 0.005139419 |\n",
      "| serial_timesteps   | 490240      |\n",
      "| time_elapsed       | 2.06e+04    |\n",
      "| total_timesteps    | 490240      |\n",
      "| value_loss         | 0.2245248   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.014791081  |\n",
      "| clipfrac           | 0.15234375   |\n",
      "| explained_variance | 0.991        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 3840         |\n",
      "| policy_entropy     | 0.76937646   |\n",
      "| policy_loss        | -0.011918202 |\n",
      "| serial_timesteps   | 491520       |\n",
      "| time_elapsed       | 2.06e+04     |\n",
      "| total_timesteps    | 491520       |\n",
      "| value_loss         | 0.6374558    |\n",
      "-------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.0050888467   |\n",
      "| clipfrac           | 0.087890625    |\n",
      "| explained_variance | 0.997          |\n",
      "| fps                | 29             |\n",
      "| n_updates          | 3850           |\n",
      "| policy_entropy     | 0.64049333     |\n",
      "| policy_loss        | -4.5049004e-05 |\n",
      "| serial_timesteps   | 492800         |\n",
      "| time_elapsed       | 2.07e+04       |\n",
      "| total_timesteps    | 492800         |\n",
      "| value_loss         | 1.2951255      |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------\n",
      "| approxkl           | 0.0029234726  |\n",
      "| clipfrac           | 0.048828125   |\n",
      "| explained_variance | 0.908         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 3860          |\n",
      "| policy_entropy     | 0.5877903     |\n",
      "| policy_loss        | -0.0021798937 |\n",
      "| serial_timesteps   | 494080        |\n",
      "| time_elapsed       | 2.07e+04      |\n",
      "| total_timesteps    | 494080        |\n",
      "| value_loss         | 0.63265795    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.008583492  |\n",
      "| clipfrac           | 0.09765625   |\n",
      "| explained_variance | 0.992        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 3870         |\n",
      "| policy_entropy     | 0.5292809    |\n",
      "| policy_loss        | -0.017655391 |\n",
      "| serial_timesteps   | 495360       |\n",
      "| time_elapsed       | 2.08e+04     |\n",
      "| total_timesteps    | 495360       |\n",
      "| value_loss         | 0.42061853   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.00031346112 |\n",
      "| clipfrac           | 0.0           |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 3880          |\n",
      "| policy_entropy     | 0.50083846    |\n",
      "| policy_loss        | -0.003387349  |\n",
      "| serial_timesteps   | 496640        |\n",
      "| time_elapsed       | 2.08e+04      |\n",
      "| total_timesteps    | 496640        |\n",
      "| value_loss         | 0.57985926    |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.009880348 |\n",
      "| clipfrac           | 0.123046875 |\n",
      "| explained_variance | 0.961       |\n",
      "| fps                | 31          |\n",
      "| n_updates          | 3890        |\n",
      "| policy_entropy     | 0.34568498  |\n",
      "| policy_loss        | 0.007688743 |\n",
      "| serial_timesteps   | 497920      |\n",
      "| time_elapsed       | 2.08e+04    |\n",
      "| total_timesteps    | 497920      |\n",
      "| value_loss         | 0.08936301  |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.011139284   |\n",
      "| clipfrac           | 0.107421875   |\n",
      "| explained_variance | 0.994         |\n",
      "| fps                | 25            |\n",
      "| n_updates          | 3900          |\n",
      "| policy_entropy     | 0.45179307    |\n",
      "| policy_loss        | -0.0049642134 |\n",
      "| serial_timesteps   | 499200        |\n",
      "| time_elapsed       | 2.09e+04      |\n",
      "| total_timesteps    | 499200        |\n",
      "| value_loss         | 0.4567529     |\n",
      "--------------------------------------\n",
      "Eval num_timesteps=500000, episode_reward=550.60 +/- 3.25\n",
      "Episode length: 589.40 +/- 1.02\n",
      "--------------------------------------\n",
      "| approxkl           | 0.021629138   |\n",
      "| clipfrac           | 0.16210938    |\n",
      "| explained_variance | 0.998         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 3910          |\n",
      "| policy_entropy     | 0.5375543     |\n",
      "| policy_loss        | -0.0039696973 |\n",
      "| serial_timesteps   | 500480        |\n",
      "| time_elapsed       | 2.1e+04       |\n",
      "| total_timesteps    | 500480        |\n",
      "| value_loss         | 0.27830678    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.00673055   |\n",
      "| clipfrac           | 0.083984375  |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 3920         |\n",
      "| policy_entropy     | 0.42296395   |\n",
      "| policy_loss        | -0.010788632 |\n",
      "| serial_timesteps   | 501760       |\n",
      "| time_elapsed       | 2.11e+04     |\n",
      "| total_timesteps    | 501760       |\n",
      "| value_loss         | 0.35699046   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.004217312  |\n",
      "| clipfrac           | 0.064453125  |\n",
      "| explained_variance | 0.97         |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 3930         |\n",
      "| policy_entropy     | 0.45257345   |\n",
      "| policy_loss        | 0.0017971704 |\n",
      "| serial_timesteps   | 503040       |\n",
      "| time_elapsed       | 2.11e+04     |\n",
      "| total_timesteps    | 503040       |\n",
      "| value_loss         | 0.09374979   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.019526357   |\n",
      "| clipfrac           | 0.22265625    |\n",
      "| explained_variance | 0.998         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 3940          |\n",
      "| policy_entropy     | 0.5665919     |\n",
      "| policy_loss        | -0.0067408388 |\n",
      "| serial_timesteps   | 504320        |\n",
      "| time_elapsed       | 2.12e+04      |\n",
      "| total_timesteps    | 504320        |\n",
      "| value_loss         | 0.14536443    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.011827035  |\n",
      "| clipfrac           | 0.12109375   |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 3950         |\n",
      "| policy_entropy     | 0.7226492    |\n",
      "| policy_loss        | -0.012871616 |\n",
      "| serial_timesteps   | 505600       |\n",
      "| time_elapsed       | 2.12e+04     |\n",
      "| total_timesteps    | 505600       |\n",
      "| value_loss         | 0.5832515    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.007985272  |\n",
      "| clipfrac           | 0.05859375   |\n",
      "| explained_variance | 0.872        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 3960         |\n",
      "| policy_entropy     | 0.37137106   |\n",
      "| policy_loss        | 0.0042297295 |\n",
      "| serial_timesteps   | 506880       |\n",
      "| time_elapsed       | 2.13e+04     |\n",
      "| total_timesteps    | 506880       |\n",
      "| value_loss         | 0.7061327    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0062684193  |\n",
      "| clipfrac           | 0.09375       |\n",
      "| explained_variance | 0.968         |\n",
      "| fps                | 26            |\n",
      "| n_updates          | 3970          |\n",
      "| policy_entropy     | 0.54958427    |\n",
      "| policy_loss        | 0.00012248149 |\n",
      "| serial_timesteps   | 508160        |\n",
      "| time_elapsed       | 2.13e+04      |\n",
      "| total_timesteps    | 508160        |\n",
      "| value_loss         | 0.68751806    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0159535    |\n",
      "| clipfrac           | 0.16601562   |\n",
      "| explained_variance | 0.994        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 3980         |\n",
      "| policy_entropy     | 0.7100068    |\n",
      "| policy_loss        | 0.0008414851 |\n",
      "| serial_timesteps   | 509440       |\n",
      "| time_elapsed       | 2.14e+04     |\n",
      "| total_timesteps    | 509440       |\n",
      "| value_loss         | 3.6887217    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0042780563 |\n",
      "| clipfrac           | 0.048828125  |\n",
      "| explained_variance | 0.907        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 3990         |\n",
      "| policy_entropy     | 0.35698548   |\n",
      "| policy_loss        | 0.006162468  |\n",
      "| serial_timesteps   | 510720       |\n",
      "| time_elapsed       | 2.14e+04     |\n",
      "| total_timesteps    | 510720       |\n",
      "| value_loss         | 0.280426     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0033246076 |\n",
      "| clipfrac           | 0.02734375   |\n",
      "| explained_variance | 0.796        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 4000         |\n",
      "| policy_entropy     | 0.55452275   |\n",
      "| policy_loss        | -0.002291222 |\n",
      "| serial_timesteps   | 512000       |\n",
      "| time_elapsed       | 2.15e+04     |\n",
      "| total_timesteps    | 512000       |\n",
      "| value_loss         | 0.7629762    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.012494093  |\n",
      "| clipfrac           | 0.140625     |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 21           |\n",
      "| n_updates          | 4010         |\n",
      "| policy_entropy     | 0.7861385    |\n",
      "| policy_loss        | -0.004620407 |\n",
      "| serial_timesteps   | 513280       |\n",
      "| time_elapsed       | 2.15e+04     |\n",
      "| total_timesteps    | 513280       |\n",
      "| value_loss         | 0.35780945   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.009198174  |\n",
      "| clipfrac           | 0.12695312   |\n",
      "| explained_variance | 0.989        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 4020         |\n",
      "| policy_entropy     | 0.61608994   |\n",
      "| policy_loss        | 0.0016557508 |\n",
      "| serial_timesteps   | 514560       |\n",
      "| time_elapsed       | 2.16e+04     |\n",
      "| total_timesteps    | 514560       |\n",
      "| value_loss         | 0.3235486    |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------\n",
      "| approxkl           | 0.03863912  |\n",
      "| clipfrac           | 0.37890625  |\n",
      "| explained_variance | 0.979       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 4030        |\n",
      "| policy_entropy     | 0.7602735   |\n",
      "| policy_loss        | 0.012356106 |\n",
      "| serial_timesteps   | 515840      |\n",
      "| time_elapsed       | 2.16e+04    |\n",
      "| total_timesteps    | 515840      |\n",
      "| value_loss         | 0.06290948  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.01142577   |\n",
      "| clipfrac           | 0.080078125  |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 4040         |\n",
      "| policy_entropy     | 0.6059415    |\n",
      "| policy_loss        | -0.013759509 |\n",
      "| serial_timesteps   | 517120       |\n",
      "| time_elapsed       | 2.17e+04     |\n",
      "| total_timesteps    | 517120       |\n",
      "| value_loss         | 0.15367043   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.010231387  |\n",
      "| clipfrac           | 0.125        |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 4050         |\n",
      "| policy_entropy     | 0.8874261    |\n",
      "| policy_loss        | -0.013700305 |\n",
      "| serial_timesteps   | 518400       |\n",
      "| time_elapsed       | 2.17e+04     |\n",
      "| total_timesteps    | 518400       |\n",
      "| value_loss         | 1.7078353    |\n",
      "-------------------------------------\n",
      "-----------------------------------\n",
      "| approxkl           | 0.02574775 |\n",
      "| clipfrac           | 0.30078125 |\n",
      "| explained_variance | 0.0421     |\n",
      "| fps                | 31         |\n",
      "| n_updates          | 4060       |\n",
      "| policy_entropy     | 0.6894355  |\n",
      "| policy_loss        | 0.01619865 |\n",
      "| serial_timesteps   | 519680     |\n",
      "| time_elapsed       | 2.18e+04   |\n",
      "| total_timesteps    | 519680     |\n",
      "| value_loss         | 10.46477   |\n",
      "-----------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0112805115 |\n",
      "| clipfrac           | 0.18945312   |\n",
      "| explained_variance | 0.768        |\n",
      "| fps                | 32           |\n",
      "| n_updates          | 4070         |\n",
      "| policy_entropy     | 0.70939577   |\n",
      "| policy_loss        | -0.010140324 |\n",
      "| serial_timesteps   | 520960       |\n",
      "| time_elapsed       | 2.18e+04     |\n",
      "| total_timesteps    | 520960       |\n",
      "| value_loss         | 133.40059    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.015788106  |\n",
      "| clipfrac           | 0.234375     |\n",
      "| explained_variance | 0.949        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 4080         |\n",
      "| policy_entropy     | 0.52118015   |\n",
      "| policy_loss        | -0.026627088 |\n",
      "| serial_timesteps   | 522240       |\n",
      "| time_elapsed       | 2.18e+04     |\n",
      "| total_timesteps    | 522240       |\n",
      "| value_loss         | 0.3735559    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.026325352  |\n",
      "| clipfrac           | 0.17773438   |\n",
      "| explained_variance | 0.839        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 4090         |\n",
      "| policy_entropy     | 0.61704457   |\n",
      "| policy_loss        | -0.011331793 |\n",
      "| serial_timesteps   | 523520       |\n",
      "| time_elapsed       | 2.19e+04     |\n",
      "| total_timesteps    | 523520       |\n",
      "| value_loss         | 1.5202937    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.021557672  |\n",
      "| clipfrac           | 0.29882812   |\n",
      "| explained_variance | 0.666        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 4100         |\n",
      "| policy_entropy     | 1.0430171    |\n",
      "| policy_loss        | 0.0024580886 |\n",
      "| serial_timesteps   | 524800       |\n",
      "| time_elapsed       | 2.19e+04     |\n",
      "| total_timesteps    | 524800       |\n",
      "| value_loss         | 58.536648    |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=525000, episode_reward=553.90 +/- 1.16\n",
      "Episode length: 594.80 +/- 1.17\n",
      "---------------------------------------\n",
      "| approxkl           | 0.023533832    |\n",
      "| clipfrac           | 0.27539062     |\n",
      "| explained_variance | 0.999          |\n",
      "| fps                | 27             |\n",
      "| n_updates          | 4110           |\n",
      "| policy_entropy     | 0.81733304     |\n",
      "| policy_loss        | -0.00065242546 |\n",
      "| serial_timesteps   | 526080         |\n",
      "| time_elapsed       | 2.21e+04       |\n",
      "| total_timesteps    | 526080         |\n",
      "| value_loss         | 0.49655104     |\n",
      "---------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.018986788  |\n",
      "| clipfrac           | 0.265625     |\n",
      "| explained_variance | 0.984        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 4120         |\n",
      "| policy_entropy     | 0.92293274   |\n",
      "| policy_loss        | 0.0021309573 |\n",
      "| serial_timesteps   | 527360       |\n",
      "| time_elapsed       | 2.21e+04     |\n",
      "| total_timesteps    | 527360       |\n",
      "| value_loss         | 0.8570048    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.045072377 |\n",
      "| clipfrac           | 0.45507812  |\n",
      "| explained_variance | 0.841       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 4130        |\n",
      "| policy_entropy     | 1.3181317   |\n",
      "| policy_loss        | 0.013918331 |\n",
      "| serial_timesteps   | 528640      |\n",
      "| time_elapsed       | 2.22e+04    |\n",
      "| total_timesteps    | 528640      |\n",
      "| value_loss         | 0.18322363  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.01365789  |\n",
      "| clipfrac           | 0.19140625  |\n",
      "| explained_variance | 0.918       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 4140        |\n",
      "| policy_entropy     | 1.1443602   |\n",
      "| policy_loss        | 0.002022567 |\n",
      "| serial_timesteps   | 529920      |\n",
      "| time_elapsed       | 2.22e+04    |\n",
      "| total_timesteps    | 529920      |\n",
      "| value_loss         | 0.34484455  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.01072509   |\n",
      "| clipfrac           | 0.14257812   |\n",
      "| explained_variance | 0.995        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 4150         |\n",
      "| policy_entropy     | 0.91999936   |\n",
      "| policy_loss        | -0.004686357 |\n",
      "| serial_timesteps   | 531200       |\n",
      "| time_elapsed       | 2.23e+04     |\n",
      "| total_timesteps    | 531200       |\n",
      "| value_loss         | 0.33176345   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0052558947  |\n",
      "| clipfrac           | 0.064453125   |\n",
      "| explained_variance | 0.993         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 4160          |\n",
      "| policy_entropy     | 0.8746558     |\n",
      "| policy_loss        | -0.0057715243 |\n",
      "| serial_timesteps   | 532480        |\n",
      "| time_elapsed       | 2.23e+04      |\n",
      "| total_timesteps    | 532480        |\n",
      "| value_loss         | 0.84998083    |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.04422026  |\n",
      "| clipfrac           | 0.32617188  |\n",
      "| explained_variance | 0.914       |\n",
      "| fps                | 31          |\n",
      "| n_updates          | 4170        |\n",
      "| policy_entropy     | 1.0992173   |\n",
      "| policy_loss        | 0.017614797 |\n",
      "| serial_timesteps   | 533760      |\n",
      "| time_elapsed       | 2.24e+04    |\n",
      "| total_timesteps    | 533760      |\n",
      "| value_loss         | 0.44565433  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.028585423  |\n",
      "| clipfrac           | 0.1640625    |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 4180         |\n",
      "| policy_entropy     | 1.12755      |\n",
      "| policy_loss        | -0.007696135 |\n",
      "| serial_timesteps   | 535040       |\n",
      "| time_elapsed       | 2.24e+04     |\n",
      "| total_timesteps    | 535040       |\n",
      "| value_loss         | 0.11745953   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.016146362  |\n",
      "| clipfrac           | 0.23828125   |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 4190         |\n",
      "| policy_entropy     | 0.961154     |\n",
      "| policy_loss        | -0.013201285 |\n",
      "| serial_timesteps   | 536320       |\n",
      "| time_elapsed       | 2.25e+04     |\n",
      "| total_timesteps    | 536320       |\n",
      "| value_loss         | 0.55493134   |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.026469802  |\n",
      "| clipfrac           | 0.45703125   |\n",
      "| explained_variance | 0.88         |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 4200         |\n",
      "| policy_entropy     | 1.2936684    |\n",
      "| policy_loss        | -0.008376591 |\n",
      "| serial_timesteps   | 537600       |\n",
      "| time_elapsed       | 2.25e+04     |\n",
      "| total_timesteps    | 537600       |\n",
      "| value_loss         | 0.13511841   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.018870391  |\n",
      "| clipfrac           | 0.296875     |\n",
      "| explained_variance | 0.995        |\n",
      "| fps                | 25           |\n",
      "| n_updates          | 4210         |\n",
      "| policy_entropy     | 1.0164047    |\n",
      "| policy_loss        | -0.018329833 |\n",
      "| serial_timesteps   | 538880       |\n",
      "| time_elapsed       | 2.26e+04     |\n",
      "| total_timesteps    | 538880       |\n",
      "| value_loss         | 0.08440298   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.012765809 |\n",
      "| clipfrac           | 0.171875    |\n",
      "| explained_variance | 0.997       |\n",
      "| fps                | 24          |\n",
      "| n_updates          | 4220        |\n",
      "| policy_entropy     | 0.7346486   |\n",
      "| policy_loss        | 0.004572856 |\n",
      "| serial_timesteps   | 540160      |\n",
      "| time_elapsed       | 2.26e+04    |\n",
      "| total_timesteps    | 540160      |\n",
      "| value_loss         | 0.7399719   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.01049298    |\n",
      "| clipfrac           | 0.095703125   |\n",
      "| explained_variance | 0.96          |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 4230          |\n",
      "| policy_entropy     | 0.42966595    |\n",
      "| policy_loss        | -0.0075524105 |\n",
      "| serial_timesteps   | 541440        |\n",
      "| time_elapsed       | 2.26e+04      |\n",
      "| total_timesteps    | 541440        |\n",
      "| value_loss         | 0.4803683     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.03419546   |\n",
      "| clipfrac           | 0.29101562   |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 4240         |\n",
      "| policy_entropy     | 0.603937     |\n",
      "| policy_loss        | 0.0031653922 |\n",
      "| serial_timesteps   | 542720       |\n",
      "| time_elapsed       | 2.27e+04     |\n",
      "| total_timesteps    | 542720       |\n",
      "| value_loss         | 0.65392196   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0017819493 |\n",
      "| clipfrac           | 0.021484375  |\n",
      "| explained_variance | 0.607        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 4250         |\n",
      "| policy_entropy     | 0.5860796    |\n",
      "| policy_loss        | 0.007663661  |\n",
      "| serial_timesteps   | 544000       |\n",
      "| time_elapsed       | 2.27e+04     |\n",
      "| total_timesteps    | 544000       |\n",
      "| value_loss         | 2.011472     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0074204253 |\n",
      "| clipfrac           | 0.1171875    |\n",
      "| explained_variance | 0.979        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 4260         |\n",
      "| policy_entropy     | 0.7276085    |\n",
      "| policy_loss        | -0.011806456 |\n",
      "| serial_timesteps   | 545280       |\n",
      "| time_elapsed       | 2.28e+04     |\n",
      "| total_timesteps    | 545280       |\n",
      "| value_loss         | 2.304117     |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.012292951   |\n",
      "| clipfrac           | 0.1640625     |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 4270          |\n",
      "| policy_entropy     | 0.8159248     |\n",
      "| policy_loss        | -0.0021736324 |\n",
      "| serial_timesteps   | 546560        |\n",
      "| time_elapsed       | 2.28e+04      |\n",
      "| total_timesteps    | 546560        |\n",
      "| value_loss         | 0.4786672     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.008025044   |\n",
      "| clipfrac           | 0.111328125   |\n",
      "| explained_variance | -0.784        |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 4280          |\n",
      "| policy_entropy     | 1.032615      |\n",
      "| policy_loss        | -0.0099536795 |\n",
      "| serial_timesteps   | 547840        |\n",
      "| time_elapsed       | 2.29e+04      |\n",
      "| total_timesteps    | 547840        |\n",
      "| value_loss         | 2.1612883     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.016297746  |\n",
      "| clipfrac           | 0.18359375   |\n",
      "| explained_variance | 0.964        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 4290         |\n",
      "| policy_entropy     | 1.0267819    |\n",
      "| policy_loss        | 0.0036648293 |\n",
      "| serial_timesteps   | 549120       |\n",
      "| time_elapsed       | 2.29e+04     |\n",
      "| total_timesteps    | 549120       |\n",
      "| value_loss         | 1.9469044    |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=550000, episode_reward=520.50 +/- 4.92\n",
      "Episode length: 585.40 +/- 1.36\n",
      "-------------------------------------\n",
      "| approxkl           | 0.016183283  |\n",
      "| clipfrac           | 0.24414062   |\n",
      "| explained_variance | 0.983        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 4300         |\n",
      "| policy_entropy     | 1.0875758    |\n",
      "| policy_loss        | 0.0010751346 |\n",
      "| serial_timesteps   | 550400       |\n",
      "| time_elapsed       | 2.31e+04     |\n",
      "| total_timesteps    | 550400       |\n",
      "| value_loss         | 0.3676434    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0140874935  |\n",
      "| clipfrac           | 0.1328125     |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 24            |\n",
      "| n_updates          | 4310          |\n",
      "| policy_entropy     | 0.982306      |\n",
      "| policy_loss        | -0.0077991844 |\n",
      "| serial_timesteps   | 551680        |\n",
      "| time_elapsed       | 2.31e+04      |\n",
      "| total_timesteps    | 551680        |\n",
      "| value_loss         | 0.61122644    |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.012000242 |\n",
      "| clipfrac           | 0.15039062  |\n",
      "| explained_variance | 0.897       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 4320        |\n",
      "| policy_entropy     | 0.6570045   |\n",
      "| policy_loss        | 0.005325219 |\n",
      "| serial_timesteps   | 552960      |\n",
      "| time_elapsed       | 2.32e+04    |\n",
      "| total_timesteps    | 552960      |\n",
      "| value_loss         | 0.34503335  |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.009911538   |\n",
      "| clipfrac           | 0.15039062    |\n",
      "| explained_variance | 0.971         |\n",
      "| fps                | 27            |\n",
      "| n_updates          | 4330          |\n",
      "| policy_entropy     | 0.74787694    |\n",
      "| policy_loss        | -0.0018108202 |\n",
      "| serial_timesteps   | 554240        |\n",
      "| time_elapsed       | 2.32e+04      |\n",
      "| total_timesteps    | 554240        |\n",
      "| value_loss         | 0.7238213     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.006917131   |\n",
      "| clipfrac           | 0.068359375   |\n",
      "| explained_variance | 0.996         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 4340          |\n",
      "| policy_entropy     | 0.92107934    |\n",
      "| policy_loss        | -0.0010034624 |\n",
      "| serial_timesteps   | 555520        |\n",
      "| time_elapsed       | 2.33e+04      |\n",
      "| total_timesteps    | 555520        |\n",
      "| value_loss         | 1.6107527     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.011208222   |\n",
      "| clipfrac           | 0.15625       |\n",
      "| explained_variance | 0.904         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 4350          |\n",
      "| policy_entropy     | 0.67046756    |\n",
      "| policy_loss        | -0.0011709542 |\n",
      "| serial_timesteps   | 556800        |\n",
      "| time_elapsed       | 2.33e+04      |\n",
      "| total_timesteps    | 556800        |\n",
      "| value_loss         | 1.2945036     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.006450644  |\n",
      "| clipfrac           | 0.076171875  |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 4360         |\n",
      "| policy_entropy     | 0.72602844   |\n",
      "| policy_loss        | -0.021411046 |\n",
      "| serial_timesteps   | 558080       |\n",
      "| time_elapsed       | 2.34e+04     |\n",
      "| total_timesteps    | 558080       |\n",
      "| value_loss         | 1.5032356    |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.010549068  |\n",
      "| clipfrac           | 0.119140625  |\n",
      "| explained_variance | 0.899        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 4370         |\n",
      "| policy_entropy     | 0.4518041    |\n",
      "| policy_loss        | 0.0064981775 |\n",
      "| serial_timesteps   | 559360       |\n",
      "| time_elapsed       | 2.34e+04     |\n",
      "| total_timesteps    | 559360       |\n",
      "| value_loss         | 0.7297934    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.07862973   |\n",
      "| clipfrac           | 0.45898438   |\n",
      "| explained_variance | 0.963        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 4380         |\n",
      "| policy_entropy     | 0.84893656   |\n",
      "| policy_loss        | -0.011584438 |\n",
      "| serial_timesteps   | 560640       |\n",
      "| time_elapsed       | 2.34e+04     |\n",
      "| total_timesteps    | 560640       |\n",
      "| value_loss         | 4.080806     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0030579763 |\n",
      "| clipfrac           | 0.048828125  |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 4390         |\n",
      "| policy_entropy     | 0.4774474    |\n",
      "| policy_loss        | -0.010299455 |\n",
      "| serial_timesteps   | 561920       |\n",
      "| time_elapsed       | 2.35e+04     |\n",
      "| total_timesteps    | 561920       |\n",
      "| value_loss         | 0.6531074    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.00947901   |\n",
      "| clipfrac           | 0.1484375    |\n",
      "| explained_variance | 0.859        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 4400         |\n",
      "| policy_entropy     | 1.0839088    |\n",
      "| policy_loss        | -0.010982623 |\n",
      "| serial_timesteps   | 563200       |\n",
      "| time_elapsed       | 2.35e+04     |\n",
      "| total_timesteps    | 563200       |\n",
      "| value_loss         | 0.2454173    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0011986147 |\n",
      "| clipfrac           | 0.00390625   |\n",
      "| explained_variance | 0.98         |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 4410         |\n",
      "| policy_entropy     | 0.39647943   |\n",
      "| policy_loss        | -0.000167466 |\n",
      "| serial_timesteps   | 564480       |\n",
      "| time_elapsed       | 2.36e+04     |\n",
      "| total_timesteps    | 564480       |\n",
      "| value_loss         | 0.5098738    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.037773736  |\n",
      "| clipfrac           | 0.27734375   |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 4420         |\n",
      "| policy_entropy     | 0.7456341    |\n",
      "| policy_loss        | -0.005316523 |\n",
      "| serial_timesteps   | 565760       |\n",
      "| time_elapsed       | 2.36e+04     |\n",
      "| total_timesteps    | 565760       |\n",
      "| value_loss         | 0.46799245   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.010769451  |\n",
      "| clipfrac           | 0.16015625   |\n",
      "| explained_variance | 0.909        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 4430         |\n",
      "| policy_entropy     | 0.4245312    |\n",
      "| policy_loss        | -0.013238921 |\n",
      "| serial_timesteps   | 567040       |\n",
      "| time_elapsed       | 2.37e+04     |\n",
      "| total_timesteps    | 567040       |\n",
      "| value_loss         | 0.18633673   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0117604835  |\n",
      "| clipfrac           | 0.123046875   |\n",
      "| explained_variance | 0.989         |\n",
      "| fps                | 27            |\n",
      "| n_updates          | 4440          |\n",
      "| policy_entropy     | 0.5127218     |\n",
      "| policy_loss        | -0.0041511827 |\n",
      "| serial_timesteps   | 568320        |\n",
      "| time_elapsed       | 2.37e+04      |\n",
      "| total_timesteps    | 568320        |\n",
      "| value_loss         | 0.2749211     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0043269163 |\n",
      "| clipfrac           | 0.0546875    |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 4450         |\n",
      "| policy_entropy     | 0.7205198    |\n",
      "| policy_loss        | -0.010394609 |\n",
      "| serial_timesteps   | 569600       |\n",
      "| time_elapsed       | 2.38e+04     |\n",
      "| total_timesteps    | 569600       |\n",
      "| value_loss         | 0.67379993   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.02417951  |\n",
      "| clipfrac           | 0.25        |\n",
      "| explained_variance | 0.977       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 4460        |\n",
      "| policy_entropy     | 0.53877884  |\n",
      "| policy_loss        | 0.008821434 |\n",
      "| serial_timesteps   | 570880      |\n",
      "| time_elapsed       | 2.38e+04    |\n",
      "| total_timesteps    | 570880      |\n",
      "| value_loss         | 0.26330093  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.017823197 |\n",
      "| clipfrac           | 0.25195312  |\n",
      "| explained_variance | 0.95        |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 4470        |\n",
      "| policy_entropy     | 0.96180695  |\n",
      "| policy_loss        | 0.004455951 |\n",
      "| serial_timesteps   | 572160      |\n",
      "| time_elapsed       | 2.39e+04    |\n",
      "| total_timesteps    | 572160      |\n",
      "| value_loss         | 0.29036275  |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.023685621   |\n",
      "| clipfrac           | 0.28710938    |\n",
      "| explained_variance | 0.993         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 4480          |\n",
      "| policy_entropy     | 0.88131046    |\n",
      "| policy_loss        | -0.0069709886 |\n",
      "| serial_timesteps   | 573440        |\n",
      "| time_elapsed       | 2.39e+04      |\n",
      "| total_timesteps    | 573440        |\n",
      "| value_loss         | 0.5823595     |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.01785093  |\n",
      "| clipfrac           | 0.1875      |\n",
      "| explained_variance | 0.999       |\n",
      "| fps                | 28          |\n",
      "| n_updates          | 4490        |\n",
      "| policy_entropy     | 0.8867785   |\n",
      "| policy_loss        | 0.008300429 |\n",
      "| serial_timesteps   | 574720      |\n",
      "| time_elapsed       | 2.4e+04     |\n",
      "| total_timesteps    | 574720      |\n",
      "| value_loss         | 0.8770516   |\n",
      "------------------------------------\n",
      "Eval num_timesteps=575000, episode_reward=548.50 +/- 1.67\n",
      "Episode length: 588.40 +/- 1.36\n",
      "-------------------------------------\n",
      "| approxkl           | 0.028689334  |\n",
      "| clipfrac           | 0.29296875   |\n",
      "| explained_variance | 0.987        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 4500         |\n",
      "| policy_entropy     | 0.9388635    |\n",
      "| policy_loss        | 0.0038598622 |\n",
      "| serial_timesteps   | 576000       |\n",
      "| time_elapsed       | 2.41e+04     |\n",
      "| total_timesteps    | 576000       |\n",
      "| value_loss         | 0.9490583    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.029665247  |\n",
      "| clipfrac           | 0.24414062   |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 4510         |\n",
      "| policy_entropy     | 0.862442     |\n",
      "| policy_loss        | -0.014543615 |\n",
      "| serial_timesteps   | 577280       |\n",
      "| time_elapsed       | 2.42e+04     |\n",
      "| total_timesteps    | 577280       |\n",
      "| value_loss         | 0.5714069    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0033306251 |\n",
      "| clipfrac           | 0.048828125  |\n",
      "| explained_variance | 0.898        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 4520         |\n",
      "| policy_entropy     | 0.70783496   |\n",
      "| policy_loss        | -0.007350714 |\n",
      "| serial_timesteps   | 578560       |\n",
      "| time_elapsed       | 2.42e+04     |\n",
      "| total_timesteps    | 578560       |\n",
      "| value_loss         | 0.25284007   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.020561403   |\n",
      "| clipfrac           | 0.2890625     |\n",
      "| explained_variance | 0.938         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 4530          |\n",
      "| policy_entropy     | 1.1620154     |\n",
      "| policy_loss        | -0.0028396281 |\n",
      "| serial_timesteps   | 579840        |\n",
      "| time_elapsed       | 2.43e+04      |\n",
      "| total_timesteps    | 579840        |\n",
      "| value_loss         | 0.54000884    |\n",
      "--------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.013189708  |\n",
      "| clipfrac           | 0.19921875   |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 4540         |\n",
      "| policy_entropy     | 0.96262854   |\n",
      "| policy_loss        | -0.019261375 |\n",
      "| serial_timesteps   | 581120       |\n",
      "| time_elapsed       | 2.43e+04     |\n",
      "| total_timesteps    | 581120       |\n",
      "| value_loss         | 0.18684134   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.031546257  |\n",
      "| clipfrac           | 0.33203125   |\n",
      "| explained_variance | 0.97         |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 4550         |\n",
      "| policy_entropy     | 0.8324605    |\n",
      "| policy_loss        | -0.013988091 |\n",
      "| serial_timesteps   | 582400       |\n",
      "| time_elapsed       | 2.43e+04     |\n",
      "| total_timesteps    | 582400       |\n",
      "| value_loss         | 0.48915654   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.019061424   |\n",
      "| clipfrac           | 0.29101562    |\n",
      "| explained_variance | 0.839         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 4560          |\n",
      "| policy_entropy     | 1.1288403     |\n",
      "| policy_loss        | -0.0019231313 |\n",
      "| serial_timesteps   | 583680        |\n",
      "| time_elapsed       | 2.44e+04      |\n",
      "| total_timesteps    | 583680        |\n",
      "| value_loss         | 0.6130989     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.021228     |\n",
      "| clipfrac           | 0.26757812   |\n",
      "| explained_variance | 0.995        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 4570         |\n",
      "| policy_entropy     | 0.99882257   |\n",
      "| policy_loss        | -0.007977134 |\n",
      "| serial_timesteps   | 584960       |\n",
      "| time_elapsed       | 2.44e+04     |\n",
      "| total_timesteps    | 584960       |\n",
      "| value_loss         | 0.14576073   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0054515996  |\n",
      "| clipfrac           | 0.064453125   |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 4580          |\n",
      "| policy_entropy     | 0.9089029     |\n",
      "| policy_loss        | -0.0059435526 |\n",
      "| serial_timesteps   | 586240        |\n",
      "| time_elapsed       | 2.45e+04      |\n",
      "| total_timesteps    | 586240        |\n",
      "| value_loss         | 0.63088506    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.007111057  |\n",
      "| clipfrac           | 0.099609375  |\n",
      "| explained_variance | 0.943        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 4590         |\n",
      "| policy_entropy     | 0.7994965    |\n",
      "| policy_loss        | 0.0005490321 |\n",
      "| serial_timesteps   | 587520       |\n",
      "| time_elapsed       | 2.45e+04     |\n",
      "| total_timesteps    | 587520       |\n",
      "| value_loss         | 0.38790014   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.028816978  |\n",
      "| clipfrac           | 0.2890625    |\n",
      "| explained_variance | 0.989        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 4600         |\n",
      "| policy_entropy     | 0.8110741    |\n",
      "| policy_loss        | 0.0060359314 |\n",
      "| serial_timesteps   | 588800       |\n",
      "| time_elapsed       | 2.46e+04     |\n",
      "| total_timesteps    | 588800       |\n",
      "| value_loss         | 0.5446554    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.007247536  |\n",
      "| clipfrac           | 0.103515625  |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 4610         |\n",
      "| policy_entropy     | 0.7325169    |\n",
      "| policy_loss        | -0.010800609 |\n",
      "| serial_timesteps   | 590080       |\n",
      "| time_elapsed       | 2.46e+04     |\n",
      "| total_timesteps    | 590080       |\n",
      "| value_loss         | 0.7288779    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.014235746  |\n",
      "| clipfrac           | 0.19140625   |\n",
      "| explained_variance | 0.334        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 4620         |\n",
      "| policy_entropy     | 0.555127     |\n",
      "| policy_loss        | 0.0023778318 |\n",
      "| serial_timesteps   | 591360       |\n",
      "| time_elapsed       | 2.47e+04     |\n",
      "| total_timesteps    | 591360       |\n",
      "| value_loss         | 1.100667     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0035640353 |\n",
      "| clipfrac           | 0.033203125  |\n",
      "| explained_variance | 0.886        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 4630         |\n",
      "| policy_entropy     | 0.4446434    |\n",
      "| policy_loss        | -0.006948227 |\n",
      "| serial_timesteps   | 592640       |\n",
      "| time_elapsed       | 2.47e+04     |\n",
      "| total_timesteps    | 592640       |\n",
      "| value_loss         | 0.6195002    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.006251255  |\n",
      "| clipfrac           | 0.072265625  |\n",
      "| explained_variance | 0.975        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 4640         |\n",
      "| policy_entropy     | 0.5164803    |\n",
      "| policy_loss        | 0.0029038633 |\n",
      "| serial_timesteps   | 593920       |\n",
      "| time_elapsed       | 2.48e+04     |\n",
      "| total_timesteps    | 593920       |\n",
      "| value_loss         | 1.1856916    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0077950754 |\n",
      "| clipfrac           | 0.1015625    |\n",
      "| explained_variance | 0.995        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 4650         |\n",
      "| policy_entropy     | 0.30385068   |\n",
      "| policy_loss        | 0.0009767214 |\n",
      "| serial_timesteps   | 595200       |\n",
      "| time_elapsed       | 2.48e+04     |\n",
      "| total_timesteps    | 595200       |\n",
      "| value_loss         | 1.1593398    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0052858805 |\n",
      "| clipfrac           | 0.076171875  |\n",
      "| explained_variance | 0.961        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 4660         |\n",
      "| policy_entropy     | 0.6192575    |\n",
      "| policy_loss        | 0.005112747  |\n",
      "| serial_timesteps   | 596480       |\n",
      "| time_elapsed       | 2.49e+04     |\n",
      "| total_timesteps    | 596480       |\n",
      "| value_loss         | 0.13885856   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0054220892  |\n",
      "| clipfrac           | 0.0703125     |\n",
      "| explained_variance | 0.992         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 4670          |\n",
      "| policy_entropy     | 0.4662742     |\n",
      "| policy_loss        | -0.0033378263 |\n",
      "| serial_timesteps   | 597760        |\n",
      "| time_elapsed       | 2.49e+04      |\n",
      "| total_timesteps    | 597760        |\n",
      "| value_loss         | 0.3074703     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.012551613  |\n",
      "| clipfrac           | 0.12109375   |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 4680         |\n",
      "| policy_entropy     | 0.57881445   |\n",
      "| policy_loss        | -0.010063905 |\n",
      "| serial_timesteps   | 599040       |\n",
      "| time_elapsed       | 2.5e+04      |\n",
      "| total_timesteps    | 599040       |\n",
      "| value_loss         | 0.29645532   |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=600000, episode_reward=541.40 +/- 2.56\n",
      "Episode length: 587.20 +/- 0.40\n",
      "-------------------------------------\n",
      "| approxkl           | 0.041797414  |\n",
      "| clipfrac           | 0.2421875    |\n",
      "| explained_variance | 0.986        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 4690         |\n",
      "| policy_entropy     | 0.36757988   |\n",
      "| policy_loss        | 0.0063193925 |\n",
      "| serial_timesteps   | 600320       |\n",
      "| time_elapsed       | 2.51e+04     |\n",
      "| total_timesteps    | 600320       |\n",
      "| value_loss         | 0.12899068   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.006382043 |\n",
      "| clipfrac           | 0.091796875 |\n",
      "| explained_variance | 0.992       |\n",
      "| fps                | 22          |\n",
      "| n_updates          | 4700        |\n",
      "| policy_entropy     | 0.46914327  |\n",
      "| policy_loss        | 0.005792101 |\n",
      "| serial_timesteps   | 601600      |\n",
      "| time_elapsed       | 2.52e+04    |\n",
      "| total_timesteps    | 601600      |\n",
      "| value_loss         | 0.42268068  |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------\n",
      "| approxkl           | 0.009321811   |\n",
      "| clipfrac           | 0.13671875    |\n",
      "| explained_variance | 0.992         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 4710          |\n",
      "| policy_entropy     | 0.7222528     |\n",
      "| policy_loss        | -0.0047454764 |\n",
      "| serial_timesteps   | 602880        |\n",
      "| time_elapsed       | 2.52e+04      |\n",
      "| total_timesteps    | 602880        |\n",
      "| value_loss         | 0.6256226     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.010367562  |\n",
      "| clipfrac           | 0.12109375   |\n",
      "| explained_variance | 0.629        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 4720         |\n",
      "| policy_entropy     | 0.34839177   |\n",
      "| policy_loss        | -0.005604347 |\n",
      "| serial_timesteps   | 604160       |\n",
      "| time_elapsed       | 2.53e+04     |\n",
      "| total_timesteps    | 604160       |\n",
      "| value_loss         | 0.8263229    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0072261808  |\n",
      "| clipfrac           | 0.115234375   |\n",
      "| explained_variance | 0.996         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 4730          |\n",
      "| policy_entropy     | 0.46926183    |\n",
      "| policy_loss        | -0.0028013168 |\n",
      "| serial_timesteps   | 605440        |\n",
      "| time_elapsed       | 2.53e+04      |\n",
      "| total_timesteps    | 605440        |\n",
      "| value_loss         | 0.7337107     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.010408941   |\n",
      "| clipfrac           | 0.13476562    |\n",
      "| explained_variance | 0.994         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 4740          |\n",
      "| policy_entropy     | 0.6311176     |\n",
      "| policy_loss        | -0.0133908745 |\n",
      "| serial_timesteps   | 606720        |\n",
      "| time_elapsed       | 2.53e+04      |\n",
      "| total_timesteps    | 606720        |\n",
      "| value_loss         | 2.2514231     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0017044418  |\n",
      "| clipfrac           | 0.021484375   |\n",
      "| explained_variance | 0.955         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 4750          |\n",
      "| policy_entropy     | 0.32193893    |\n",
      "| policy_loss        | 3.3908174e-05 |\n",
      "| serial_timesteps   | 608000        |\n",
      "| time_elapsed       | 2.54e+04      |\n",
      "| total_timesteps    | 608000        |\n",
      "| value_loss         | 0.06376368    |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.02254437  |\n",
      "| clipfrac           | 0.20507812  |\n",
      "| explained_variance | 0.998       |\n",
      "| fps                | 25          |\n",
      "| n_updates          | 4760        |\n",
      "| policy_entropy     | 0.4024356   |\n",
      "| policy_loss        | 0.006948119 |\n",
      "| serial_timesteps   | 609280      |\n",
      "| time_elapsed       | 2.54e+04    |\n",
      "| total_timesteps    | 609280      |\n",
      "| value_loss         | 0.10363611  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.02434845   |\n",
      "| clipfrac           | 0.234375     |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 4770         |\n",
      "| policy_entropy     | 0.68431276   |\n",
      "| policy_loss        | -0.011147622 |\n",
      "| serial_timesteps   | 610560       |\n",
      "| time_elapsed       | 2.55e+04     |\n",
      "| total_timesteps    | 610560       |\n",
      "| value_loss         | 1.7788261    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.046211302   |\n",
      "| clipfrac           | 0.22070312    |\n",
      "| explained_variance | 0.948         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 4780          |\n",
      "| policy_entropy     | 0.68871933    |\n",
      "| policy_loss        | -0.0059163403 |\n",
      "| serial_timesteps   | 611840        |\n",
      "| time_elapsed       | 2.55e+04      |\n",
      "| total_timesteps    | 611840        |\n",
      "| value_loss         | 0.11926935    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.016606051   |\n",
      "| clipfrac           | 0.15039062    |\n",
      "| explained_variance | 0.993         |\n",
      "| fps                | 26            |\n",
      "| n_updates          | 4790          |\n",
      "| policy_entropy     | 0.6515215     |\n",
      "| policy_loss        | -0.0044927592 |\n",
      "| serial_timesteps   | 613120        |\n",
      "| time_elapsed       | 2.56e+04      |\n",
      "| total_timesteps    | 613120        |\n",
      "| value_loss         | 0.12008777    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0066530486 |\n",
      "| clipfrac           | 0.080078125  |\n",
      "| explained_variance | 0.993        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 4800         |\n",
      "| policy_entropy     | 0.6198677    |\n",
      "| policy_loss        | -0.01192299  |\n",
      "| serial_timesteps   | 614400       |\n",
      "| time_elapsed       | 2.56e+04     |\n",
      "| total_timesteps    | 614400       |\n",
      "| value_loss         | 1.9231325    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.01352969   |\n",
      "| clipfrac           | 0.16601562   |\n",
      "| explained_variance | 0.892        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 4810         |\n",
      "| policy_entropy     | 0.5949293    |\n",
      "| policy_loss        | 0.0038181767 |\n",
      "| serial_timesteps   | 615680       |\n",
      "| time_elapsed       | 2.57e+04     |\n",
      "| total_timesteps    | 615680       |\n",
      "| value_loss         | 0.118396655  |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0103555685 |\n",
      "| clipfrac           | 0.171875     |\n",
      "| explained_variance | 0.723        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 4820         |\n",
      "| policy_entropy     | 1.1677727    |\n",
      "| policy_loss        | -0.005420968 |\n",
      "| serial_timesteps   | 616960       |\n",
      "| time_elapsed       | 2.57e+04     |\n",
      "| total_timesteps    | 616960       |\n",
      "| value_loss         | 1.4737666    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.01698932   |\n",
      "| clipfrac           | 0.23046875   |\n",
      "| explained_variance | 0.988        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 4830         |\n",
      "| policy_entropy     | 0.8936304    |\n",
      "| policy_loss        | -0.010528987 |\n",
      "| serial_timesteps   | 618240       |\n",
      "| time_elapsed       | 2.58e+04     |\n",
      "| total_timesteps    | 618240       |\n",
      "| value_loss         | 2.1586964    |\n",
      "-------------------------------------\n",
      "-----------------------------------\n",
      "| approxkl           | 0.01882496 |\n",
      "| clipfrac           | 0.23632812 |\n",
      "| explained_variance | 0.976      |\n",
      "| fps                | 30         |\n",
      "| n_updates          | 4840       |\n",
      "| policy_entropy     | 0.94829535 |\n",
      "| policy_loss        | 0.01755643 |\n",
      "| serial_timesteps   | 619520     |\n",
      "| time_elapsed       | 2.58e+04   |\n",
      "| total_timesteps    | 619520     |\n",
      "| value_loss         | 0.2851914  |\n",
      "-----------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.008508854   |\n",
      "| clipfrac           | 0.091796875   |\n",
      "| explained_variance | 0.695         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 4850          |\n",
      "| policy_entropy     | 0.54521644    |\n",
      "| policy_loss        | -0.0036223019 |\n",
      "| serial_timesteps   | 620800        |\n",
      "| time_elapsed       | 2.59e+04      |\n",
      "| total_timesteps    | 620800        |\n",
      "| value_loss         | 0.58533156    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0052182656 |\n",
      "| clipfrac           | 0.0703125    |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 4860         |\n",
      "| policy_entropy     | 0.50529724   |\n",
      "| policy_loss        | -0.011026761 |\n",
      "| serial_timesteps   | 622080       |\n",
      "| time_elapsed       | 2.59e+04     |\n",
      "| total_timesteps    | 622080       |\n",
      "| value_loss         | 0.32652998   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.019175872 |\n",
      "| clipfrac           | 0.21875     |\n",
      "| explained_variance | 0.995       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 4870        |\n",
      "| policy_entropy     | 0.55601895  |\n",
      "| policy_loss        | 0.005010587 |\n",
      "| serial_timesteps   | 623360      |\n",
      "| time_elapsed       | 2.6e+04     |\n",
      "| total_timesteps    | 623360      |\n",
      "| value_loss         | 0.44847348  |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.0106338905 |\n",
      "| clipfrac           | 0.12890625   |\n",
      "| explained_variance | 0.871        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 4880         |\n",
      "| policy_entropy     | 0.4665639    |\n",
      "| policy_loss        | -0.004355187 |\n",
      "| serial_timesteps   | 624640       |\n",
      "| time_elapsed       | 2.6e+04      |\n",
      "| total_timesteps    | 624640       |\n",
      "| value_loss         | 0.27314812   |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=625000, episode_reward=548.90 +/- 0.58\n",
      "Episode length: 584.60 +/- 0.49\n",
      "------------------------------------\n",
      "| approxkl           | 0.005257166 |\n",
      "| clipfrac           | 0.076171875 |\n",
      "| explained_variance | 0.988       |\n",
      "| fps                | 27          |\n",
      "| n_updates          | 4890        |\n",
      "| policy_entropy     | 0.38051936  |\n",
      "| policy_loss        | 0.002719936 |\n",
      "| serial_timesteps   | 625920      |\n",
      "| time_elapsed       | 2.62e+04    |\n",
      "| total_timesteps    | 625920      |\n",
      "| value_loss         | 0.25106397  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.024955997  |\n",
      "| clipfrac           | 0.2109375    |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 4900         |\n",
      "| policy_entropy     | 0.68082166   |\n",
      "| policy_loss        | -0.023061356 |\n",
      "| serial_timesteps   | 627200       |\n",
      "| time_elapsed       | 2.62e+04     |\n",
      "| total_timesteps    | 627200       |\n",
      "| value_loss         | 0.16031604   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.038223367 |\n",
      "| clipfrac           | 0.32226562  |\n",
      "| explained_variance | 0.886       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 4910        |\n",
      "| policy_entropy     | 0.58856595  |\n",
      "| policy_loss        | 0.009904936 |\n",
      "| serial_timesteps   | 628480      |\n",
      "| time_elapsed       | 2.62e+04    |\n",
      "| total_timesteps    | 628480      |\n",
      "| value_loss         | 0.3064798   |\n",
      "------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.0070440858   |\n",
      "| clipfrac           | 0.109375       |\n",
      "| explained_variance | 0.983          |\n",
      "| fps                | 30             |\n",
      "| n_updates          | 4920           |\n",
      "| policy_entropy     | 0.46429414     |\n",
      "| policy_loss        | -0.00025467807 |\n",
      "| serial_timesteps   | 629760         |\n",
      "| time_elapsed       | 2.63e+04       |\n",
      "| total_timesteps    | 629760         |\n",
      "| value_loss         | 0.1333085      |\n",
      "---------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.011072997   |\n",
      "| clipfrac           | 0.091796875   |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 4930          |\n",
      "| policy_entropy     | 0.5564081     |\n",
      "| policy_loss        | -0.0031687024 |\n",
      "| serial_timesteps   | 631040        |\n",
      "| time_elapsed       | 2.63e+04      |\n",
      "| total_timesteps    | 631040        |\n",
      "| value_loss         | 0.16722713    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.008530847  |\n",
      "| clipfrac           | 0.119140625  |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 4940         |\n",
      "| policy_entropy     | 0.48924404   |\n",
      "| policy_loss        | 0.0039268835 |\n",
      "| serial_timesteps   | 632320       |\n",
      "| time_elapsed       | 2.64e+04     |\n",
      "| total_timesteps    | 632320       |\n",
      "| value_loss         | 0.37095514   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.014750789  |\n",
      "| clipfrac           | 0.19726562   |\n",
      "| explained_variance | 0.953        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 4950         |\n",
      "| policy_entropy     | 0.56283134   |\n",
      "| policy_loss        | 0.0095028095 |\n",
      "| serial_timesteps   | 633600       |\n",
      "| time_elapsed       | 2.64e+04     |\n",
      "| total_timesteps    | 633600       |\n",
      "| value_loss         | 0.12492609   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0046085934 |\n",
      "| clipfrac           | 0.072265625  |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 4960         |\n",
      "| policy_entropy     | 0.4826535    |\n",
      "| policy_loss        | -0.005266615 |\n",
      "| serial_timesteps   | 634880       |\n",
      "| time_elapsed       | 2.65e+04     |\n",
      "| total_timesteps    | 634880       |\n",
      "| value_loss         | 0.30967006   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.006683914  |\n",
      "| clipfrac           | 0.0859375    |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 4970         |\n",
      "| policy_entropy     | 0.52014744   |\n",
      "| policy_loss        | -0.007477808 |\n",
      "| serial_timesteps   | 636160       |\n",
      "| time_elapsed       | 2.65e+04     |\n",
      "| total_timesteps    | 636160       |\n",
      "| value_loss         | 0.45239857   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.045296255  |\n",
      "| clipfrac           | 0.34179688   |\n",
      "| explained_variance | 0.248        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 4980         |\n",
      "| policy_entropy     | 0.56761855   |\n",
      "| policy_loss        | 0.0064496044 |\n",
      "| serial_timesteps   | 637440       |\n",
      "| time_elapsed       | 2.66e+04     |\n",
      "| total_timesteps    | 637440       |\n",
      "| value_loss         | 0.6987369    |\n",
      "-------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.021727182    |\n",
      "| clipfrac           | 0.16015625     |\n",
      "| explained_variance | 0.998          |\n",
      "| fps                | 24             |\n",
      "| n_updates          | 4990           |\n",
      "| policy_entropy     | 0.5770078      |\n",
      "| policy_loss        | -0.00026656128 |\n",
      "| serial_timesteps   | 638720         |\n",
      "| time_elapsed       | 2.66e+04       |\n",
      "| total_timesteps    | 638720         |\n",
      "| value_loss         | 0.073923364    |\n",
      "---------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.008997699  |\n",
      "| clipfrac           | 0.087890625  |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 5000         |\n",
      "| policy_entropy     | 0.6361185    |\n",
      "| policy_loss        | -0.012598229 |\n",
      "| serial_timesteps   | 640000       |\n",
      "| time_elapsed       | 2.67e+04     |\n",
      "| total_timesteps    | 640000       |\n",
      "| value_loss         | 3.7532165    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.013520215  |\n",
      "| clipfrac           | 0.16992188   |\n",
      "| explained_variance | 0.869        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 5010         |\n",
      "| policy_entropy     | 0.7541266    |\n",
      "| policy_loss        | -0.007647913 |\n",
      "| serial_timesteps   | 641280       |\n",
      "| time_elapsed       | 2.67e+04     |\n",
      "| total_timesteps    | 641280       |\n",
      "| value_loss         | 0.2011402    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.003941513   |\n",
      "| clipfrac           | 0.060546875   |\n",
      "| explained_variance | 0.996         |\n",
      "| fps                | 25            |\n",
      "| n_updates          | 5020          |\n",
      "| policy_entropy     | 0.40373844    |\n",
      "| policy_loss        | -0.0019074337 |\n",
      "| serial_timesteps   | 642560        |\n",
      "| time_elapsed       | 2.68e+04      |\n",
      "| total_timesteps    | 642560        |\n",
      "| value_loss         | 0.11439418    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.010048499   |\n",
      "| clipfrac           | 0.13085938    |\n",
      "| explained_variance | 0.998         |\n",
      "| fps                | 25            |\n",
      "| n_updates          | 5030          |\n",
      "| policy_entropy     | 0.5454281     |\n",
      "| policy_loss        | -0.0052998485 |\n",
      "| serial_timesteps   | 643840        |\n",
      "| time_elapsed       | 2.68e+04      |\n",
      "| total_timesteps    | 643840        |\n",
      "| value_loss         | 0.78027034    |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.01686787  |\n",
      "| clipfrac           | 0.17773438  |\n",
      "| explained_variance | 0.548       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 5040        |\n",
      "| policy_entropy     | 0.4402133   |\n",
      "| policy_loss        | -0.01526664 |\n",
      "| serial_timesteps   | 645120      |\n",
      "| time_elapsed       | 2.69e+04    |\n",
      "| total_timesteps    | 645120      |\n",
      "| value_loss         | 0.20846143  |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------\n",
      "| approxkl           | 0.014965226 |\n",
      "| clipfrac           | 0.1328125   |\n",
      "| explained_variance | 0.99        |\n",
      "| fps                | 28          |\n",
      "| n_updates          | 5050        |\n",
      "| policy_entropy     | 0.25472295  |\n",
      "| policy_loss        | 0.005533169 |\n",
      "| serial_timesteps   | 646400      |\n",
      "| time_elapsed       | 2.69e+04    |\n",
      "| total_timesteps    | 646400      |\n",
      "| value_loss         | 0.11644608  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.007809319  |\n",
      "| clipfrac           | 0.1171875    |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 5060         |\n",
      "| policy_entropy     | 0.6538745    |\n",
      "| policy_loss        | -0.011584206 |\n",
      "| serial_timesteps   | 647680       |\n",
      "| time_elapsed       | 2.7e+04      |\n",
      "| total_timesteps    | 647680       |\n",
      "| value_loss         | 0.15162387   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.040072553  |\n",
      "| clipfrac           | 0.25         |\n",
      "| explained_variance | 0.981        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 5070         |\n",
      "| policy_entropy     | 0.33635387   |\n",
      "| policy_loss        | 0.0041043833 |\n",
      "| serial_timesteps   | 648960       |\n",
      "| time_elapsed       | 2.7e+04      |\n",
      "| total_timesteps    | 648960       |\n",
      "| value_loss         | 0.13529716   |\n",
      "-------------------------------------\n",
      "Eval num_timesteps=650000, episode_reward=554.20 +/- 1.21\n",
      "Episode length: 588.80 +/- 0.98\n",
      "--------------------------------------\n",
      "| approxkl           | 0.01578696    |\n",
      "| clipfrac           | 0.1328125     |\n",
      "| explained_variance | 0.952         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 5080          |\n",
      "| policy_entropy     | 0.22854933    |\n",
      "| policy_loss        | -0.0020667957 |\n",
      "| serial_timesteps   | 650240        |\n",
      "| time_elapsed       | 2.72e+04      |\n",
      "| total_timesteps    | 650240        |\n",
      "| value_loss         | 0.24928826    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.018493095  |\n",
      "| clipfrac           | 0.13085938   |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 5090         |\n",
      "| policy_entropy     | 0.26154318   |\n",
      "| policy_loss        | -0.010200847 |\n",
      "| serial_timesteps   | 651520       |\n",
      "| time_elapsed       | 2.72e+04     |\n",
      "| total_timesteps    | 651520       |\n",
      "| value_loss         | 0.28385583   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.021376563  |\n",
      "| clipfrac           | 0.21289062   |\n",
      "| explained_variance | 0.995        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 5100         |\n",
      "| policy_entropy     | 0.58571714   |\n",
      "| policy_loss        | -0.025433524 |\n",
      "| serial_timesteps   | 652800       |\n",
      "| time_elapsed       | 2.72e+04     |\n",
      "| total_timesteps    | 652800       |\n",
      "| value_loss         | 1.6167142    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.014526593 |\n",
      "| clipfrac           | 0.14648438  |\n",
      "| explained_variance | 0.519       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 5110        |\n",
      "| policy_entropy     | 0.29809847  |\n",
      "| policy_loss        | 0.012392241 |\n",
      "| serial_timesteps   | 654080      |\n",
      "| time_elapsed       | 2.73e+04    |\n",
      "| total_timesteps    | 654080      |\n",
      "| value_loss         | 0.14183117  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0023413408 |\n",
      "| clipfrac           | 0.025390625  |\n",
      "| explained_variance | 0.984        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 5120         |\n",
      "| policy_entropy     | 0.24178194   |\n",
      "| policy_loss        | 0.0021800417 |\n",
      "| serial_timesteps   | 655360       |\n",
      "| time_elapsed       | 2.73e+04     |\n",
      "| total_timesteps    | 655360       |\n",
      "| value_loss         | 0.19251855   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.011534928  |\n",
      "| clipfrac           | 0.1015625    |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 5130         |\n",
      "| policy_entropy     | 0.51481557   |\n",
      "| policy_loss        | -0.014540247 |\n",
      "| serial_timesteps   | 656640       |\n",
      "| time_elapsed       | 2.74e+04     |\n",
      "| total_timesteps    | 656640       |\n",
      "| value_loss         | 0.13573214   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.013647306  |\n",
      "| clipfrac           | 0.140625     |\n",
      "| explained_variance | 0.985        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 5140         |\n",
      "| policy_entropy     | 0.28954777   |\n",
      "| policy_loss        | 0.0075612385 |\n",
      "| serial_timesteps   | 657920       |\n",
      "| time_elapsed       | 2.74e+04     |\n",
      "| total_timesteps    | 657920       |\n",
      "| value_loss         | 0.103059374  |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.009180195  |\n",
      "| clipfrac           | 0.099609375  |\n",
      "| explained_variance | 0.89         |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 5150         |\n",
      "| policy_entropy     | 0.35895357   |\n",
      "| policy_loss        | 0.0072469334 |\n",
      "| serial_timesteps   | 659200       |\n",
      "| time_elapsed       | 2.75e+04     |\n",
      "| total_timesteps    | 659200       |\n",
      "| value_loss         | 0.6128742    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.013914113  |\n",
      "| clipfrac           | 0.091796875  |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 5160         |\n",
      "| policy_entropy     | 0.41647547   |\n",
      "| policy_loss        | 0.0016643228 |\n",
      "| serial_timesteps   | 660480       |\n",
      "| time_elapsed       | 2.75e+04     |\n",
      "| total_timesteps    | 660480       |\n",
      "| value_loss         | 0.3605668    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0016712648 |\n",
      "| clipfrac           | 0.029296875  |\n",
      "| explained_variance | 0.987        |\n",
      "| fps                | 29           |\n",
      "| n_updates          | 5170         |\n",
      "| policy_entropy     | 0.26769057   |\n",
      "| policy_loss        | 0.001255516  |\n",
      "| serial_timesteps   | 661760       |\n",
      "| time_elapsed       | 2.76e+04     |\n",
      "| total_timesteps    | 661760       |\n",
      "| value_loss         | 1.3021083    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.015172449   |\n",
      "| clipfrac           | 0.2421875     |\n",
      "| explained_variance | 0.976         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 5180          |\n",
      "| policy_entropy     | 0.69809705    |\n",
      "| policy_loss        | -0.0023257877 |\n",
      "| serial_timesteps   | 663040        |\n",
      "| time_elapsed       | 2.76e+04      |\n",
      "| total_timesteps    | 663040        |\n",
      "| value_loss         | 0.1347447     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.018193739   |\n",
      "| clipfrac           | 0.09765625    |\n",
      "| explained_variance | 0.995         |\n",
      "| fps                | 23            |\n",
      "| n_updates          | 5190          |\n",
      "| policy_entropy     | 0.25870514    |\n",
      "| policy_loss        | -0.0031148046 |\n",
      "| serial_timesteps   | 664320        |\n",
      "| time_elapsed       | 2.77e+04      |\n",
      "| total_timesteps    | 664320        |\n",
      "| value_loss         | 0.50034356    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.031868316  |\n",
      "| clipfrac           | 0.22265625   |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 5200         |\n",
      "| policy_entropy     | 0.7448499    |\n",
      "| policy_loss        | -0.013015497 |\n",
      "| serial_timesteps   | 665600       |\n",
      "| time_elapsed       | 2.77e+04     |\n",
      "| total_timesteps    | 665600       |\n",
      "| value_loss         | 2.913525     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.002197002  |\n",
      "| clipfrac           | 0.01953125   |\n",
      "| explained_variance | 0.944        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 5210         |\n",
      "| policy_entropy     | 0.36968493   |\n",
      "| policy_loss        | 0.0023825727 |\n",
      "| serial_timesteps   | 666880       |\n",
      "| time_elapsed       | 2.78e+04     |\n",
      "| total_timesteps    | 666880       |\n",
      "| value_loss         | 0.18918227   |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.01298795   |\n",
      "| clipfrac           | 0.111328125  |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 5220         |\n",
      "| policy_entropy     | 0.33483627   |\n",
      "| policy_loss        | -0.006508298 |\n",
      "| serial_timesteps   | 668160       |\n",
      "| time_elapsed       | 2.78e+04     |\n",
      "| total_timesteps    | 668160       |\n",
      "| value_loss         | 0.11728486   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0041061784  |\n",
      "| clipfrac           | 0.0546875     |\n",
      "| explained_variance | 0.996         |\n",
      "| fps                | 27            |\n",
      "| n_updates          | 5230          |\n",
      "| policy_entropy     | 0.4459505     |\n",
      "| policy_loss        | -0.0030512048 |\n",
      "| serial_timesteps   | 669440        |\n",
      "| time_elapsed       | 2.79e+04      |\n",
      "| total_timesteps    | 669440        |\n",
      "| value_loss         | 2.4187746     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.007425786   |\n",
      "| clipfrac           | 0.083984375   |\n",
      "| explained_variance | 0.948         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 5240          |\n",
      "| policy_entropy     | 0.78219616    |\n",
      "| policy_loss        | -0.0034987137 |\n",
      "| serial_timesteps   | 670720        |\n",
      "| time_elapsed       | 2.79e+04      |\n",
      "| total_timesteps    | 670720        |\n",
      "| value_loss         | 0.10537046    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0077110976 |\n",
      "| clipfrac           | 0.103515625  |\n",
      "| explained_variance | 0.998        |\n",
      "| fps                | 25           |\n",
      "| n_updates          | 5250         |\n",
      "| policy_entropy     | 0.705942     |\n",
      "| policy_loss        | 0.0012091574 |\n",
      "| serial_timesteps   | 672000       |\n",
      "| time_elapsed       | 2.8e+04      |\n",
      "| total_timesteps    | 672000       |\n",
      "| value_loss         | 0.16256285   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0075443517  |\n",
      "| clipfrac           | 0.095703125   |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 25            |\n",
      "| n_updates          | 5260          |\n",
      "| policy_entropy     | 0.42062414    |\n",
      "| policy_loss        | -0.0017183498 |\n",
      "| serial_timesteps   | 673280        |\n",
      "| time_elapsed       | 2.8e+04       |\n",
      "| total_timesteps    | 673280        |\n",
      "| value_loss         | 0.43607908    |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.021284124 |\n",
      "| clipfrac           | 0.25976562  |\n",
      "| explained_variance | 0.94        |\n",
      "| fps                | 31          |\n",
      "| n_updates          | 5270        |\n",
      "| policy_entropy     | 0.6839037   |\n",
      "| policy_loss        | 0.013229826 |\n",
      "| serial_timesteps   | 674560      |\n",
      "| time_elapsed       | 2.8e+04     |\n",
      "| total_timesteps    | 674560      |\n",
      "| value_loss         | 0.2327094   |\n",
      "------------------------------------\n",
      "Eval num_timesteps=675000, episode_reward=548.00 +/- 2.86\n",
      "Episode length: 586.80 +/- 0.98\n",
      "--------------------------------------\n",
      "| approxkl           | 0.020421525   |\n",
      "| clipfrac           | 0.21875       |\n",
      "| explained_variance | 0.983         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 5280          |\n",
      "| policy_entropy     | 0.52184206    |\n",
      "| policy_loss        | -0.0024758026 |\n",
      "| serial_timesteps   | 675840        |\n",
      "| time_elapsed       | 2.82e+04      |\n",
      "| total_timesteps    | 675840        |\n",
      "| value_loss         | 0.11721597    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.007039262  |\n",
      "| clipfrac           | 0.099609375  |\n",
      "| explained_variance | 0.995        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 5290         |\n",
      "| policy_entropy     | 0.42580682   |\n",
      "| policy_loss        | -0.012330996 |\n",
      "| serial_timesteps   | 677120       |\n",
      "| time_elapsed       | 2.82e+04     |\n",
      "| total_timesteps    | 677120       |\n",
      "| value_loss         | 0.122815095  |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.005298798   |\n",
      "| clipfrac           | 0.076171875   |\n",
      "| explained_variance | 0.998         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 5300          |\n",
      "| policy_entropy     | 0.3976688     |\n",
      "| policy_loss        | -0.0041162083 |\n",
      "| serial_timesteps   | 678400        |\n",
      "| time_elapsed       | 2.83e+04      |\n",
      "| total_timesteps    | 678400        |\n",
      "| value_loss         | 1.0264758     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0060587125  |\n",
      "| clipfrac           | 0.0390625     |\n",
      "| explained_variance | 0.965         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 5310          |\n",
      "| policy_entropy     | 0.41712272    |\n",
      "| policy_loss        | -0.0046655578 |\n",
      "| serial_timesteps   | 679680        |\n",
      "| time_elapsed       | 2.83e+04      |\n",
      "| total_timesteps    | 679680        |\n",
      "| value_loss         | 0.045765307   |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0090692425 |\n",
      "| clipfrac           | 0.09765625   |\n",
      "| explained_variance | 0.936        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 5320         |\n",
      "| policy_entropy     | 0.3822223    |\n",
      "| policy_loss        | 0.0037444856 |\n",
      "| serial_timesteps   | 680960       |\n",
      "| time_elapsed       | 2.84e+04     |\n",
      "| total_timesteps    | 680960       |\n",
      "| value_loss         | 7.902988     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.004349428  |\n",
      "| clipfrac           | 0.068359375  |\n",
      "| explained_variance | 1            |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 5330         |\n",
      "| policy_entropy     | 0.39536342   |\n",
      "| policy_loss        | -0.005659702 |\n",
      "| serial_timesteps   | 682240       |\n",
      "| time_elapsed       | 2.84e+04     |\n",
      "| total_timesteps    | 682240       |\n",
      "| value_loss         | 0.46081522   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.005031986  |\n",
      "| clipfrac           | 0.0703125    |\n",
      "| explained_variance | 0.944        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 5340         |\n",
      "| policy_entropy     | 0.44832784   |\n",
      "| policy_loss        | 0.0037040398 |\n",
      "| serial_timesteps   | 683520       |\n",
      "| time_elapsed       | 2.85e+04     |\n",
      "| total_timesteps    | 683520       |\n",
      "| value_loss         | 0.3334023    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0037030128 |\n",
      "| clipfrac           | 0.05859375   |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 5350         |\n",
      "| policy_entropy     | 0.5339556    |\n",
      "| policy_loss        | -0.011919171 |\n",
      "| serial_timesteps   | 684800       |\n",
      "| time_elapsed       | 2.85e+04     |\n",
      "| total_timesteps    | 684800       |\n",
      "| value_loss         | 0.15057078   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.066261634  |\n",
      "| clipfrac           | 0.14648438   |\n",
      "| explained_variance | 1            |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 5360         |\n",
      "| policy_entropy     | 0.1976422    |\n",
      "| policy_loss        | -0.016681386 |\n",
      "| serial_timesteps   | 686080       |\n",
      "| time_elapsed       | 2.86e+04     |\n",
      "| total_timesteps    | 686080       |\n",
      "| value_loss         | 0.23266122   |\n",
      "-------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.00913146     |\n",
      "| clipfrac           | 0.123046875    |\n",
      "| explained_variance | 0.914          |\n",
      "| fps                | 31             |\n",
      "| n_updates          | 5370           |\n",
      "| policy_entropy     | 0.48525378     |\n",
      "| policy_loss        | -0.00046830997 |\n",
      "| serial_timesteps   | 687360         |\n",
      "| time_elapsed       | 2.86e+04       |\n",
      "| total_timesteps    | 687360         |\n",
      "| value_loss         | 0.19499849     |\n",
      "---------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.013484566   |\n",
      "| clipfrac           | 0.15820312    |\n",
      "| explained_variance | 0.996         |\n",
      "| fps                | 26            |\n",
      "| n_updates          | 5380          |\n",
      "| policy_entropy     | 0.6265435     |\n",
      "| policy_loss        | 0.00020654243 |\n",
      "| serial_timesteps   | 688640        |\n",
      "| time_elapsed       | 2.87e+04      |\n",
      "| total_timesteps    | 688640        |\n",
      "| value_loss         | 0.08623349    |\n",
      "--------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.015643148  |\n",
      "| clipfrac           | 0.1015625    |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 5390         |\n",
      "| policy_entropy     | 0.2842043    |\n",
      "| policy_loss        | 0.0037033574 |\n",
      "| serial_timesteps   | 689920       |\n",
      "| time_elapsed       | 2.87e+04     |\n",
      "| total_timesteps    | 689920       |\n",
      "| value_loss         | 0.28174672   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0064677475 |\n",
      "| clipfrac           | 0.109375     |\n",
      "| explained_variance | 0.723        |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 5400         |\n",
      "| policy_entropy     | 0.31746432   |\n",
      "| policy_loss        | 0.008327364  |\n",
      "| serial_timesteps   | 691200       |\n",
      "| time_elapsed       | 2.88e+04     |\n",
      "| total_timesteps    | 691200       |\n",
      "| value_loss         | 0.08557816   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.041164003 |\n",
      "| clipfrac           | 0.21289062  |\n",
      "| explained_variance | 0.993       |\n",
      "| fps                | 28          |\n",
      "| n_updates          | 5410        |\n",
      "| policy_entropy     | 0.38535157  |\n",
      "| policy_loss        | 0.019386144 |\n",
      "| serial_timesteps   | 692480      |\n",
      "| time_elapsed       | 2.88e+04    |\n",
      "| total_timesteps    | 692480      |\n",
      "| value_loss         | 0.13309117  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.013956111 |\n",
      "| clipfrac           | 0.068359375 |\n",
      "| explained_variance | 0.847       |\n",
      "| fps                | 21          |\n",
      "| n_updates          | 5420        |\n",
      "| policy_entropy     | 0.24841073  |\n",
      "| policy_loss        | 0.002016924 |\n",
      "| serial_timesteps   | 693760      |\n",
      "| time_elapsed       | 2.89e+04    |\n",
      "| total_timesteps    | 693760      |\n",
      "| value_loss         | 15.065989   |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0022561485  |\n",
      "| clipfrac           | 0.01953125    |\n",
      "| explained_variance | 0.989         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 5430          |\n",
      "| policy_entropy     | 0.44787997    |\n",
      "| policy_loss        | -0.0026149584 |\n",
      "| serial_timesteps   | 695040        |\n",
      "| time_elapsed       | 2.89e+04      |\n",
      "| total_timesteps    | 695040        |\n",
      "| value_loss         | 0.23463286    |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.011748665 |\n",
      "| clipfrac           | 0.099609375 |\n",
      "| explained_variance | 0.962       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 5440        |\n",
      "| policy_entropy     | 0.3164182   |\n",
      "| policy_loss        | -0.00202756 |\n",
      "| serial_timesteps   | 696320      |\n",
      "| time_elapsed       | 2.89e+04    |\n",
      "| total_timesteps    | 696320      |\n",
      "| value_loss         | 0.27505288  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0084874965 |\n",
      "| clipfrac           | 0.09375      |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 5450         |\n",
      "| policy_entropy     | 0.38330945   |\n",
      "| policy_loss        | 0.005023403  |\n",
      "| serial_timesteps   | 697600       |\n",
      "| time_elapsed       | 2.9e+04      |\n",
      "| total_timesteps    | 697600       |\n",
      "| value_loss         | 0.36248335   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.015009074   |\n",
      "| clipfrac           | 0.080078125   |\n",
      "| explained_variance | 0.997         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 5460          |\n",
      "| policy_entropy     | 0.2731848     |\n",
      "| policy_loss        | -0.0034494353 |\n",
      "| serial_timesteps   | 698880        |\n",
      "| time_elapsed       | 2.9e+04       |\n",
      "| total_timesteps    | 698880        |\n",
      "| value_loss         | 1.2125342     |\n",
      "--------------------------------------\n",
      "Eval num_timesteps=700000, episode_reward=555.90 +/- 1.24\n",
      "Episode length: 592.60 +/- 1.02\n",
      "-------------------------------------\n",
      "| approxkl           | 0.01280838   |\n",
      "| clipfrac           | 0.12109375   |\n",
      "| explained_variance | 0.974        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 5470         |\n",
      "| policy_entropy     | 0.33571902   |\n",
      "| policy_loss        | 0.0020086877 |\n",
      "| serial_timesteps   | 700160       |\n",
      "| time_elapsed       | 2.92e+04     |\n",
      "| total_timesteps    | 700160       |\n",
      "| value_loss         | 0.48433512   |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.052637804 |\n",
      "| clipfrac           | 0.2109375   |\n",
      "| explained_variance | 0.983       |\n",
      "| fps                | 29          |\n",
      "| n_updates          | 5480        |\n",
      "| policy_entropy     | 0.31148428  |\n",
      "| policy_loss        | 0.008327705 |\n",
      "| serial_timesteps   | 701440      |\n",
      "| time_elapsed       | 2.92e+04    |\n",
      "| total_timesteps    | 701440      |\n",
      "| value_loss         | 0.12590995  |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.039702177  |\n",
      "| clipfrac           | 0.1171875    |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 5490         |\n",
      "| policy_entropy     | 0.32237205   |\n",
      "| policy_loss        | -0.012069244 |\n",
      "| serial_timesteps   | 702720       |\n",
      "| time_elapsed       | 2.93e+04     |\n",
      "| total_timesteps    | 702720       |\n",
      "| value_loss         | 0.6643828    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.021503843 |\n",
      "| clipfrac           | 0.2109375   |\n",
      "| explained_variance | 0.995       |\n",
      "| fps                | 30          |\n",
      "| n_updates          | 5500        |\n",
      "| policy_entropy     | 0.43842125  |\n",
      "| policy_loss        | 0.008493631 |\n",
      "| serial_timesteps   | 704000      |\n",
      "| time_elapsed       | 2.93e+04    |\n",
      "| total_timesteps    | 704000      |\n",
      "| value_loss         | 0.6741384   |\n",
      "------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.006853559  |\n",
      "| clipfrac           | 0.091796875  |\n",
      "| explained_variance | 0.99         |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 5510         |\n",
      "| policy_entropy     | 0.46916857   |\n",
      "| policy_loss        | -0.004146885 |\n",
      "| serial_timesteps   | 705280       |\n",
      "| time_elapsed       | 2.94e+04     |\n",
      "| total_timesteps    | 705280       |\n",
      "| value_loss         | 0.056068085  |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.028049897 |\n",
      "| clipfrac           | 0.14453125  |\n",
      "| explained_variance | 0.995       |\n",
      "| fps                | 23          |\n",
      "| n_updates          | 5520        |\n",
      "| policy_entropy     | 0.3881704   |\n",
      "| policy_loss        | 0.006518037 |\n",
      "| serial_timesteps   | 706560      |\n",
      "| time_elapsed       | 2.94e+04    |\n",
      "| total_timesteps    | 706560      |\n",
      "| value_loss         | 0.15068574  |\n",
      "------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.05484806    |\n",
      "| clipfrac           | 0.1953125     |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 5530          |\n",
      "| policy_entropy     | 0.33343816    |\n",
      "| policy_loss        | -0.0065512904 |\n",
      "| serial_timesteps   | 707840        |\n",
      "| time_elapsed       | 2.95e+04      |\n",
      "| total_timesteps    | 707840        |\n",
      "| value_loss         | 0.4208185     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.012267748   |\n",
      "| clipfrac           | 0.21679688    |\n",
      "| explained_variance | 0.97          |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 5540          |\n",
      "| policy_entropy     | 0.8628063     |\n",
      "| policy_loss        | -0.0061507067 |\n",
      "| serial_timesteps   | 709120        |\n",
      "| time_elapsed       | 2.95e+04      |\n",
      "| total_timesteps    | 709120        |\n",
      "| value_loss         | 0.052460827   |\n",
      "--------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.017995931 |\n",
      "| clipfrac           | 0.24023438  |\n",
      "| explained_variance | 0.996       |\n",
      "| fps                | 25          |\n",
      "| n_updates          | 5550        |\n",
      "| policy_entropy     | 0.6273901   |\n",
      "| policy_loss        | 0.010444952 |\n",
      "| serial_timesteps   | 710400      |\n",
      "| time_elapsed       | 2.96e+04    |\n",
      "| total_timesteps    | 710400      |\n",
      "| value_loss         | 0.49734625  |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------\n",
      "| approxkl           | 0.011167748   |\n",
      "| clipfrac           | 0.08203125    |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 25            |\n",
      "| n_updates          | 5560          |\n",
      "| policy_entropy     | 0.38382417    |\n",
      "| policy_loss        | -0.0032306924 |\n",
      "| serial_timesteps   | 711680        |\n",
      "| time_elapsed       | 2.96e+04      |\n",
      "| total_timesteps    | 711680        |\n",
      "| value_loss         | 1.4406863     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0113394745  |\n",
      "| clipfrac           | 0.0546875     |\n",
      "| explained_variance | 0.666         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 5570          |\n",
      "| policy_entropy     | 0.6033982     |\n",
      "| policy_loss        | -0.0026750022 |\n",
      "| serial_timesteps   | 712960        |\n",
      "| time_elapsed       | 2.97e+04      |\n",
      "| total_timesteps    | 712960        |\n",
      "| value_loss         | 1.2567248     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0035285891 |\n",
      "| clipfrac           | 0.041015625  |\n",
      "| explained_variance | 0.995        |\n",
      "| fps                | 25           |\n",
      "| n_updates          | 5580         |\n",
      "| policy_entropy     | 0.18243897   |\n",
      "| policy_loss        | -0.009350453 |\n",
      "| serial_timesteps   | 714240       |\n",
      "| time_elapsed       | 2.97e+04     |\n",
      "| total_timesteps    | 714240       |\n",
      "| value_loss         | 0.31637862   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.009488614  |\n",
      "| clipfrac           | 0.13476562   |\n",
      "| explained_variance | 0.994        |\n",
      "| fps                | 26           |\n",
      "| n_updates          | 5590         |\n",
      "| policy_entropy     | 0.32674783   |\n",
      "| policy_loss        | -0.004755714 |\n",
      "| serial_timesteps   | 715520       |\n",
      "| time_elapsed       | 2.98e+04     |\n",
      "| total_timesteps    | 715520       |\n",
      "| value_loss         | 2.630396     |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.00702337    |\n",
      "| clipfrac           | 0.091796875   |\n",
      "| explained_variance | 0.949         |\n",
      "| fps                | 31            |\n",
      "| n_updates          | 5600          |\n",
      "| policy_entropy     | 0.5515446     |\n",
      "| policy_loss        | -0.0073163435 |\n",
      "| serial_timesteps   | 716800        |\n",
      "| time_elapsed       | 2.98e+04      |\n",
      "| total_timesteps    | 716800        |\n",
      "| value_loss         | 0.12631674    |\n",
      "--------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.009620032    |\n",
      "| clipfrac           | 0.12109375     |\n",
      "| explained_variance | 0.995          |\n",
      "| fps                | 25             |\n",
      "| n_updates          | 5610           |\n",
      "| policy_entropy     | 0.32975996     |\n",
      "| policy_loss        | -2.1490734e-05 |\n",
      "| serial_timesteps   | 718080         |\n",
      "| time_elapsed       | 2.99e+04       |\n",
      "| total_timesteps    | 718080         |\n",
      "| value_loss         | 0.16485733     |\n",
      "---------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0069832383  |\n",
      "| clipfrac           | 0.083984375   |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 24            |\n",
      "| n_updates          | 5620          |\n",
      "| policy_entropy     | 0.56461257    |\n",
      "| policy_loss        | -0.0040020626 |\n",
      "| serial_timesteps   | 719360        |\n",
      "| time_elapsed       | 2.99e+04      |\n",
      "| total_timesteps    | 719360        |\n",
      "| value_loss         | 0.7409071     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.006510233   |\n",
      "| clipfrac           | 0.06640625    |\n",
      "| explained_variance | 0.908         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 5630          |\n",
      "| policy_entropy     | 0.493838      |\n",
      "| policy_loss        | -0.0045158206 |\n",
      "| serial_timesteps   | 720640        |\n",
      "| time_elapsed       | 2.99e+04      |\n",
      "| total_timesteps    | 720640        |\n",
      "| value_loss         | 0.16826376    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0074401638  |\n",
      "| clipfrac           | 0.095703125   |\n",
      "| explained_variance | 0.993         |\n",
      "| fps                | 27            |\n",
      "| n_updates          | 5640          |\n",
      "| policy_entropy     | 0.69762534    |\n",
      "| policy_loss        | -0.0015806938 |\n",
      "| serial_timesteps   | 721920        |\n",
      "| time_elapsed       | 3e+04         |\n",
      "| total_timesteps    | 721920        |\n",
      "| value_loss         | 0.18207861    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.018010192  |\n",
      "| clipfrac           | 0.125        |\n",
      "| explained_variance | 0.999        |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 5650         |\n",
      "| policy_entropy     | 0.473232     |\n",
      "| policy_loss        | 0.0051869988 |\n",
      "| serial_timesteps   | 723200       |\n",
      "| time_elapsed       | 3e+04        |\n",
      "| total_timesteps    | 723200       |\n",
      "| value_loss         | 0.22724022   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.017546287   |\n",
      "| clipfrac           | 0.27539062    |\n",
      "| explained_variance | 0.912         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 5660          |\n",
      "| policy_entropy     | 0.6341166     |\n",
      "| policy_loss        | -0.0033535413 |\n",
      "| serial_timesteps   | 724480        |\n",
      "| time_elapsed       | 3.01e+04      |\n",
      "| total_timesteps    | 724480        |\n",
      "| value_loss         | 0.03940926    |\n",
      "--------------------------------------\n",
      "Eval num_timesteps=725000, episode_reward=550.40 +/- 0.86\n",
      "Episode length: 585.40 +/- 1.02\n",
      "-------------------------------------\n",
      "| approxkl           | 0.01068355   |\n",
      "| clipfrac           | 0.16601562   |\n",
      "| explained_variance | 0.916        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 5670         |\n",
      "| policy_entropy     | 0.55783355   |\n",
      "| policy_loss        | -0.013633443 |\n",
      "| serial_timesteps   | 725760       |\n",
      "| time_elapsed       | 3.02e+04     |\n",
      "| total_timesteps    | 725760       |\n",
      "| value_loss         | 0.12918408   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0064290687 |\n",
      "| clipfrac           | 0.078125     |\n",
      "| explained_variance | 0.992        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 5680         |\n",
      "| policy_entropy     | 0.2956629    |\n",
      "| policy_loss        | -0.002355427 |\n",
      "| serial_timesteps   | 727040       |\n",
      "| time_elapsed       | 3.03e+04     |\n",
      "| total_timesteps    | 727040       |\n",
      "| value_loss         | 0.28551224   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.004179583   |\n",
      "| clipfrac           | 0.060546875   |\n",
      "| explained_variance | 0.998         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 5690          |\n",
      "| policy_entropy     | 0.31146067    |\n",
      "| policy_loss        | -0.0031730803 |\n",
      "| serial_timesteps   | 728320        |\n",
      "| time_elapsed       | 3.03e+04      |\n",
      "| total_timesteps    | 728320        |\n",
      "| value_loss         | 0.7117551     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0055018947  |\n",
      "| clipfrac           | 0.078125      |\n",
      "| explained_variance | 0.797         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 5700          |\n",
      "| policy_entropy     | 0.5084407     |\n",
      "| policy_loss        | -0.0060838023 |\n",
      "| serial_timesteps   | 729600        |\n",
      "| time_elapsed       | 3.04e+04      |\n",
      "| total_timesteps    | 729600        |\n",
      "| value_loss         | 0.20170036    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.04050107   |\n",
      "| clipfrac           | 0.1796875    |\n",
      "| explained_variance | 0.986        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 5710         |\n",
      "| policy_entropy     | 0.22304413   |\n",
      "| policy_loss        | -0.018162431 |\n",
      "| serial_timesteps   | 730880       |\n",
      "| time_elapsed       | 3.04e+04     |\n",
      "| total_timesteps    | 730880       |\n",
      "| value_loss         | 0.16566373   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.018368803   |\n",
      "| clipfrac           | 0.125         |\n",
      "| explained_variance | 0.998         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 5720          |\n",
      "| policy_entropy     | 0.31186974    |\n",
      "| policy_loss        | -0.0150061175 |\n",
      "| serial_timesteps   | 732160        |\n",
      "| time_elapsed       | 3.05e+04      |\n",
      "| total_timesteps    | 732160        |\n",
      "| value_loss         | 1.1169056     |\n",
      "--------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.008610224  |\n",
      "| clipfrac           | 0.123046875  |\n",
      "| explained_variance | 0.78         |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 5730         |\n",
      "| policy_entropy     | 0.54402494   |\n",
      "| policy_loss        | 0.0012298812 |\n",
      "| serial_timesteps   | 733440       |\n",
      "| time_elapsed       | 3.05e+04     |\n",
      "| total_timesteps    | 733440       |\n",
      "| value_loss         | 0.20898114   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.007679379  |\n",
      "| clipfrac           | 0.0859375    |\n",
      "| explained_variance | 0.904        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 5740         |\n",
      "| policy_entropy     | 0.31632793   |\n",
      "| policy_loss        | -0.007904394 |\n",
      "| serial_timesteps   | 734720       |\n",
      "| time_elapsed       | 3.06e+04     |\n",
      "| total_timesteps    | 734720       |\n",
      "| value_loss         | 0.6030319    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0054024924 |\n",
      "| clipfrac           | 0.0625       |\n",
      "| explained_variance | 0.997        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 5750         |\n",
      "| policy_entropy     | 0.36812216   |\n",
      "| policy_loss        | -0.007606866 |\n",
      "| serial_timesteps   | 736000       |\n",
      "| time_elapsed       | 3.06e+04     |\n",
      "| total_timesteps    | 736000       |\n",
      "| value_loss         | 0.98203355   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.009334944   |\n",
      "| clipfrac           | 0.111328125   |\n",
      "| explained_variance | 0.783         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 5760          |\n",
      "| policy_entropy     | 0.59689397    |\n",
      "| policy_loss        | 2.1012966e-06 |\n",
      "| serial_timesteps   | 737280        |\n",
      "| time_elapsed       | 3.07e+04      |\n",
      "| total_timesteps    | 737280        |\n",
      "| value_loss         | 0.25096685    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0065367566 |\n",
      "| clipfrac           | 0.072265625  |\n",
      "| explained_variance | 0.883        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 5770         |\n",
      "| policy_entropy     | 0.42208436   |\n",
      "| policy_loss        | 0.0037648224 |\n",
      "| serial_timesteps   | 738560       |\n",
      "| time_elapsed       | 3.07e+04     |\n",
      "| total_timesteps    | 738560       |\n",
      "| value_loss         | 0.46721932   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.009916848   |\n",
      "| clipfrac           | 0.14648438    |\n",
      "| explained_variance | 0.999         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 5780          |\n",
      "| policy_entropy     | 0.48226878    |\n",
      "| policy_loss        | 0.00085080997 |\n",
      "| serial_timesteps   | 739840        |\n",
      "| time_elapsed       | 3.08e+04      |\n",
      "| total_timesteps    | 739840        |\n",
      "| value_loss         | 0.21410266    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.008859428  |\n",
      "| clipfrac           | 0.15625      |\n",
      "| explained_variance | 0.948        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 5790         |\n",
      "| policy_entropy     | 0.7173048    |\n",
      "| policy_loss        | -0.005666312 |\n",
      "| serial_timesteps   | 741120       |\n",
      "| time_elapsed       | 3.08e+04     |\n",
      "| total_timesteps    | 741120       |\n",
      "| value_loss         | 0.32652912   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0066757165 |\n",
      "| clipfrac           | 0.09765625   |\n",
      "| explained_variance | 0.977        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 5800         |\n",
      "| policy_entropy     | 0.44470155   |\n",
      "| policy_loss        | 0.004807382  |\n",
      "| serial_timesteps   | 742400       |\n",
      "| time_elapsed       | 3.09e+04     |\n",
      "| total_timesteps    | 742400       |\n",
      "| value_loss         | 0.2455606    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.009042915  |\n",
      "| clipfrac           | 0.091796875  |\n",
      "| explained_variance | 0.996        |\n",
      "| fps                | 22           |\n",
      "| n_updates          | 5810         |\n",
      "| policy_entropy     | 0.43907395   |\n",
      "| policy_loss        | -0.010477828 |\n",
      "| serial_timesteps   | 743680       |\n",
      "| time_elapsed       | 3.09e+04     |\n",
      "| total_timesteps    | 743680       |\n",
      "| value_loss         | 0.8237057    |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0050114887  |\n",
      "| clipfrac           | 0.041015625   |\n",
      "| explained_variance | 0.861         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 5820          |\n",
      "| policy_entropy     | 0.70675695    |\n",
      "| policy_loss        | 0.00021110737 |\n",
      "| serial_timesteps   | 744960        |\n",
      "| time_elapsed       | 3.09e+04      |\n",
      "| total_timesteps    | 744960        |\n",
      "| value_loss         | 0.26823092    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.003927045   |\n",
      "| clipfrac           | 0.056640625   |\n",
      "| explained_variance | 0.965         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 5830          |\n",
      "| policy_entropy     | 0.3030453     |\n",
      "| policy_loss        | -0.0019691347 |\n",
      "| serial_timesteps   | 746240        |\n",
      "| time_elapsed       | 3.1e+04       |\n",
      "| total_timesteps    | 746240        |\n",
      "| value_loss         | 0.18037584    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.0043747975 |\n",
      "| clipfrac           | 0.060546875  |\n",
      "| explained_variance | 0.138        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 5840         |\n",
      "| policy_entropy     | 0.30022472   |\n",
      "| policy_loss        | -0.007065052 |\n",
      "| serial_timesteps   | 747520       |\n",
      "| time_elapsed       | 3.1e+04      |\n",
      "| total_timesteps    | 747520       |\n",
      "| value_loss         | 4.763486     |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0009815859  |\n",
      "| clipfrac           | 0.0078125     |\n",
      "| explained_variance | 0.793         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 5850          |\n",
      "| policy_entropy     | 0.3480432     |\n",
      "| policy_loss        | -0.0017311219 |\n",
      "| serial_timesteps   | 748800        |\n",
      "| time_elapsed       | 3.11e+04      |\n",
      "| total_timesteps    | 748800        |\n",
      "| value_loss         | 1.8189076     |\n",
      "--------------------------------------\n",
      "Eval num_timesteps=750000, episode_reward=526.10 +/- 1.62\n",
      "Episode length: 588.80 +/- 1.17\n",
      "--------------------------------------\n",
      "| approxkl           | 0.01312701    |\n",
      "| clipfrac           | 0.087890625   |\n",
      "| explained_variance | 0.757         |\n",
      "| fps                | 1             |\n",
      "| n_updates          | 5860          |\n",
      "| policy_entropy     | 0.30987906    |\n",
      "| policy_loss        | -0.0010787679 |\n",
      "| serial_timesteps   | 750080        |\n",
      "| time_elapsed       | 3.11e+04      |\n",
      "| total_timesteps    | 750080        |\n",
      "| value_loss         | 4.505746      |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.06876909   |\n",
      "| clipfrac           | 0.27734375   |\n",
      "| explained_variance | 0.0549       |\n",
      "| fps                | 31           |\n",
      "| n_updates          | 5870         |\n",
      "| policy_entropy     | 0.41676417   |\n",
      "| policy_loss        | -0.007666316 |\n",
      "| serial_timesteps   | 751360       |\n",
      "| time_elapsed       | 3.13e+04     |\n",
      "| total_timesteps    | 751360       |\n",
      "| value_loss         | 1.6046344    |\n",
      "-------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.02663355  |\n",
      "| clipfrac           | 0.20507812  |\n",
      "| explained_variance | 0.995       |\n",
      "| fps                | 26          |\n",
      "| n_updates          | 5880        |\n",
      "| policy_entropy     | 0.57118714  |\n",
      "| policy_loss        | 0.010872014 |\n",
      "| serial_timesteps   | 752640      |\n",
      "| time_elapsed       | 3.13e+04    |\n",
      "| total_timesteps    | 752640      |\n",
      "| value_loss         | 0.36635694  |\n",
      "------------------------------------\n",
      "------------------------------------\n",
      "| approxkl           | 0.014193311 |\n",
      "| clipfrac           | 0.12890625  |\n",
      "| explained_variance | 0.999       |\n",
      "| fps                | 24          |\n",
      "| n_updates          | 5890        |\n",
      "| policy_entropy     | 0.3626064   |\n",
      "| policy_loss        | -0.01548791 |\n",
      "| serial_timesteps   | 753920      |\n",
      "| time_elapsed       | 3.14e+04    |\n",
      "| total_timesteps    | 753920      |\n",
      "| value_loss         | 0.5439277   |\n",
      "------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "| approxkl           | 0.025860284  |\n",
      "| clipfrac           | 0.23828125   |\n",
      "| explained_variance | 0.848        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 5900         |\n",
      "| policy_entropy     | 0.79812205   |\n",
      "| policy_loss        | 0.0034988949 |\n",
      "| serial_timesteps   | 755200       |\n",
      "| time_elapsed       | 3.14e+04     |\n",
      "| total_timesteps    | 755200       |\n",
      "| value_loss         | 0.12588803   |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.003558382  |\n",
      "| clipfrac           | 0.048828125  |\n",
      "| explained_variance | 0.99         |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 5910         |\n",
      "| policy_entropy     | 0.26257      |\n",
      "| policy_loss        | 0.0018060072 |\n",
      "| serial_timesteps   | 756480       |\n",
      "| time_elapsed       | 3.15e+04     |\n",
      "| total_timesteps    | 756480       |\n",
      "| value_loss         | 0.34458894   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.003356753   |\n",
      "| clipfrac           | 0.044921875   |\n",
      "| explained_variance | 0.952         |\n",
      "| fps                | 28            |\n",
      "| n_updates          | 5920          |\n",
      "| policy_entropy     | 0.42638358    |\n",
      "| policy_loss        | -8.115056e-05 |\n",
      "| serial_timesteps   | 757760        |\n",
      "| time_elapsed       | 3.15e+04      |\n",
      "| total_timesteps    | 757760        |\n",
      "| value_loss         | 23.381433     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0058354205  |\n",
      "| clipfrac           | 0.056640625   |\n",
      "| explained_variance | 0.992         |\n",
      "| fps                | 25            |\n",
      "| n_updates          | 5930          |\n",
      "| policy_entropy     | 0.21484858    |\n",
      "| policy_loss        | -0.0042377166 |\n",
      "| serial_timesteps   | 759040        |\n",
      "| time_elapsed       | 3.16e+04      |\n",
      "| total_timesteps    | 759040        |\n",
      "| value_loss         | 0.32910824    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.02323439    |\n",
      "| clipfrac           | 0.140625      |\n",
      "| explained_variance | 0.828         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 5940          |\n",
      "| policy_entropy     | 0.44862023    |\n",
      "| policy_loss        | -0.0080657285 |\n",
      "| serial_timesteps   | 760320        |\n",
      "| time_elapsed       | 3.16e+04      |\n",
      "| total_timesteps    | 760320        |\n",
      "| value_loss         | 107.80002     |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.013734118  |\n",
      "| clipfrac           | 0.16601562   |\n",
      "| explained_variance | 0.968        |\n",
      "| fps                | 25           |\n",
      "| n_updates          | 5950         |\n",
      "| policy_entropy     | 0.7842635    |\n",
      "| policy_loss        | -0.009722466 |\n",
      "| serial_timesteps   | 761600       |\n",
      "| time_elapsed       | 3.16e+04     |\n",
      "| total_timesteps    | 761600       |\n",
      "| value_loss         | 35.935104    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.015045651  |\n",
      "| clipfrac           | 0.23242188   |\n",
      "| explained_variance | 0.979        |\n",
      "| fps                | 27           |\n",
      "| n_updates          | 5960         |\n",
      "| policy_entropy     | 0.53957856   |\n",
      "| policy_loss        | -0.008194402 |\n",
      "| serial_timesteps   | 762880       |\n",
      "| time_elapsed       | 3.17e+04     |\n",
      "| total_timesteps    | 762880       |\n",
      "| value_loss         | 5.082975     |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.028655497  |\n",
      "| clipfrac           | 0.16796875   |\n",
      "| explained_variance | 0.99         |\n",
      "| fps                | 23           |\n",
      "| n_updates          | 5970         |\n",
      "| policy_entropy     | 0.3439925    |\n",
      "| policy_loss        | -0.015952943 |\n",
      "| serial_timesteps   | 764160       |\n",
      "| time_elapsed       | 3.17e+04     |\n",
      "| total_timesteps    | 764160       |\n",
      "| value_loss         | 5.6832676    |\n",
      "-------------------------------------\n",
      "---------------------------------------\n",
      "| approxkl           | 0.00965955     |\n",
      "| clipfrac           | 0.13085938     |\n",
      "| explained_variance | 0.975          |\n",
      "| fps                | 30             |\n",
      "| n_updates          | 5980           |\n",
      "| policy_entropy     | 0.545611       |\n",
      "| policy_loss        | -0.00068379706 |\n",
      "| serial_timesteps   | 765440         |\n",
      "| time_elapsed       | 3.18e+04       |\n",
      "| total_timesteps    | 765440         |\n",
      "| value_loss         | 0.59124017     |\n",
      "---------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.005222391  |\n",
      "| clipfrac           | 0.076171875  |\n",
      "| explained_variance | 0.984        |\n",
      "| fps                | 28           |\n",
      "| n_updates          | 5990         |\n",
      "| policy_entropy     | 0.26110876   |\n",
      "| policy_loss        | -0.007963744 |\n",
      "| serial_timesteps   | 766720       |\n",
      "| time_elapsed       | 3.18e+04     |\n",
      "| total_timesteps    | 766720       |\n",
      "| value_loss         | 0.47759023   |\n",
      "-------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.0040727654  |\n",
      "| clipfrac           | 0.05078125    |\n",
      "| explained_variance | 0.993         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 6000          |\n",
      "| policy_entropy     | 0.3826645     |\n",
      "| policy_loss        | -0.0047376794 |\n",
      "| serial_timesteps   | 768000        |\n",
      "| time_elapsed       | 3.19e+04      |\n",
      "| total_timesteps    | 768000        |\n",
      "| value_loss         | 1.1130986     |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.011233442   |\n",
      "| clipfrac           | 0.1484375     |\n",
      "| explained_variance | 0.953         |\n",
      "| fps                | 30            |\n",
      "| n_updates          | 6010          |\n",
      "| policy_entropy     | 0.4193814     |\n",
      "| policy_loss        | -0.0007627418 |\n",
      "| serial_timesteps   | 769280        |\n",
      "| time_elapsed       | 3.19e+04      |\n",
      "| total_timesteps    | 769280        |\n",
      "| value_loss         | 0.13028105    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.014587612   |\n",
      "| clipfrac           | 0.13476562    |\n",
      "| explained_variance | 0.945         |\n",
      "| fps                | 29            |\n",
      "| n_updates          | 6020          |\n",
      "| policy_entropy     | 0.36759245    |\n",
      "| policy_loss        | -0.0005670851 |\n",
      "| serial_timesteps   | 770560        |\n",
      "| time_elapsed       | 3.2e+04       |\n",
      "| total_timesteps    | 770560        |\n",
      "| value_loss         | 0.32940117    |\n",
      "--------------------------------------\n",
      "--------------------------------------\n",
      "| approxkl           | 0.023703547   |\n",
      "| clipfrac           | 0.16210938    |\n",
      "| explained_variance | 0.998         |\n",
      "| fps                | 22            |\n",
      "| n_updates          | 6030          |\n",
      "| policy_entropy     | 0.38173366    |\n",
      "| policy_loss        | -0.0054028886 |\n",
      "| serial_timesteps   | 771840        |\n",
      "| time_elapsed       | 3.2e+04       |\n",
      "| total_timesteps    | 771840        |\n",
      "| value_loss         | 0.45119876    |\n",
      "--------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.012700102  |\n",
      "| clipfrac           | 0.13476562   |\n",
      "| explained_variance | 0.988        |\n",
      "| fps                | 30           |\n",
      "| n_updates          | 6040         |\n",
      "| policy_entropy     | 0.59403735   |\n",
      "| policy_loss        | -0.007867393 |\n",
      "| serial_timesteps   | 773120       |\n",
      "| time_elapsed       | 3.21e+04     |\n",
      "| total_timesteps    | 773120       |\n",
      "| value_loss         | 0.7133992    |\n",
      "-------------------------------------\n",
      "-------------------------------------\n",
      "| approxkl           | 0.001468015  |\n",
      "| clipfrac           | 0.025390625  |\n",
      "| explained_variance | 0.977        |\n",
      "| fps                | 24           |\n",
      "| n_updates          | 6050         |\n",
      "| policy_entropy     | 0.24486096   |\n",
      "| policy_loss        | -0.004321963 |\n",
      "| serial_timesteps   | 774400       |\n",
      "| time_elapsed       | 3.21e+04     |\n",
      "| total_timesteps    | 774400       |\n",
      "| value_loss         | 0.33970058   |\n",
      "-------------------------------------\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-0228918fb3de>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     44\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     45\u001b[0m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mPPO2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCnnPolicy\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtensorboard_log\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtensorboard_dir\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 46\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtotal_timesteps\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m2000000\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlog_interval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     47\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     48\u001b[0m \u001b[0mobs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\ppo2\\ppo2.py\u001b[0m in \u001b[0;36mlearn\u001b[1;34m(self, total_timesteps, callback, log_interval, tb_log_name, reset_num_timesteps)\u001b[0m\n\u001b[0;32m    334\u001b[0m                 \u001b[0mcallback\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_rollout_start\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    335\u001b[0m                 \u001b[1;31m# true_reward is the reward without discount\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 336\u001b[1;33m                 \u001b[0mrollout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrunner\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcallback\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    337\u001b[0m                 \u001b[1;31m# Unpack\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    338\u001b[0m                 \u001b[0mobs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreturns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmasks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mactions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mneglogpacs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstates\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mep_infos\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrue_reward\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrollout\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\runners.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, callback)\u001b[0m\n\u001b[0;32m     46\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallback\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     47\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontinue_training\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 48\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_run\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     49\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     50\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\ppo2\\ppo2.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    486\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallback\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    487\u001b[0m                 \u001b[1;31m# Abort training early\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 488\u001b[1;33m                 \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallback\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_step\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    489\u001b[0m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontinue_training\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    490\u001b[0m                     \u001b[1;31m# Return dummy values\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\callbacks.py\u001b[0m in \u001b[0;36mon_step\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     87\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_timesteps\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_timesteps\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     88\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 89\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_on_step\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     90\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mon_training_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\callbacks.py\u001b[0m in \u001b[0;36m_on_step\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    163\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    164\u001b[0m             \u001b[1;31m# Return False (stop training) if at least one callback returns False\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 165\u001b[1;33m             \u001b[0mcontinue_training\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_step\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mcontinue_training\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    166\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mcontinue_training\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    167\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\callbacks.py\u001b[0m in \u001b[0;36mon_step\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     87\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_timesteps\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_timesteps\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     88\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 89\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_on_step\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     90\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mon_training_end\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\callbacks.py\u001b[0m in \u001b[0;36m_on_step\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    293\u001b[0m                                                                \u001b[0mrender\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    294\u001b[0m                                                                \u001b[0mdeterministic\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdeterministic\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 295\u001b[1;33m                                                                return_episode_rewards=True)\n\u001b[0m\u001b[0;32m    296\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    297\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlog_path\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\evaluation.py\u001b[0m in \u001b[0;36mevaluate_policy\u001b[1;34m(model, env, n_eval_episodes, deterministic, render, callback, reward_threshold, return_episode_rewards)\u001b[0m\n\u001b[0;32m     37\u001b[0m         \u001b[1;32mwhile\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     38\u001b[0m             \u001b[0maction\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mstate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdeterministic\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 39\u001b[1;33m             \u001b[0mobs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_info\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0menv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maction\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     40\u001b[0m             \u001b[0mepisode_reward\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mreward\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     41\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mcallback\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\vec_env\\base_vec_env.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self, actions)\u001b[0m\n\u001b[0;32m    148\u001b[0m         \"\"\"\n\u001b[0;32m    149\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mactions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 150\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep_wait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    151\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    152\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget_images\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[0mSequence\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\stable_baselines\\common\\vec_env\\dummy_vec_env.py\u001b[0m in \u001b[0;36mstep_wait\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     40\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0menv_idx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_envs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     41\u001b[0m             \u001b[0mobs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuf_rews\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0menv_idx\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuf_dones\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0menv_idx\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuf_infos\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0menv_idx\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 42\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0menvs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0menv_idx\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mactions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0menv_idx\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     43\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbuf_dones\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0menv_idx\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     44\u001b[0m                 \u001b[1;31m# save final observation where user can get it, then reset\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\research\\AirSim-RL\\gym_airsim\\airsim_car_env.py\u001b[0m in \u001b[0;36mstep\u001b[1;34m(self, action)\u001b[0m\n\u001b[0;32m     55\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     56\u001b[0m         \u001b[1;31m# get observation\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 57\u001b[1;33m         \u001b[0mobservation\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcar_agent\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mobserve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     58\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mobservation\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minfo\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\research\\AirSim-RL\\gym_airsim\\car_agent.py\u001b[0m in \u001b[0;36mobserve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     41\u001b[0m         \u001b[1;32mwhile\u001b[0m \u001b[0msize\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimage_size\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;31m# Sometimes simGetImages() return an unexpected resonpse.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m                                        \u001b[1;31m# If so, try it again.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 43\u001b[1;33m             \u001b[0mresponse\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msimGetImages\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mImageRequest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mImageType\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mScene\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     44\u001b[0m             \u001b[0mimg1d_rgb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrombuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimage_data_uint8\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0muint8\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     45\u001b[0m             \u001b[0msize\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mimg1d_rgb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\airsim\\client.py\u001b[0m in \u001b[0;36msimGetImages\u001b[1;34m(self, requests, vehicle_name)\u001b[0m\n\u001b[0;32m    101\u001b[0m     \u001b[1;31m# image_type uses one of the ImageType members\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    102\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0msimGetImages\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrequests\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvehicle_name\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m''\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 103\u001b[1;33m         \u001b[0mresponses_raw\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclient\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'simGetImages'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrequests\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvehicle_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    104\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mImageResponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfrom_msgpack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse_raw\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mresponse_raw\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mresponses_raw\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    105\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\msgpackrpc\\session.py\u001b[0m in \u001b[0;36mcall\u001b[1;34m(self, method, *args)\u001b[0m\n\u001b[0;32m     39\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 41\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msend_request\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     42\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     43\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcall_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\msgpackrpc\\future.py\u001b[0m in \u001b[0;36mget\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     26\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     27\u001b[0m         \u001b[1;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_set_flag\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\msgpackrpc\\future.py\u001b[0m in \u001b[0;36mjoin\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     20\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mjoin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     21\u001b[0m         \u001b[1;32mwhile\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_set_flag\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 22\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_loop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\msgpackrpc\\loop.py\u001b[0m in \u001b[0;36mstart\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     20\u001b[0m         \"\"\"\n\u001b[0;32m     21\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 22\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_ioloop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     23\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mstop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tornado\\ioloop.py\u001b[0m in \u001b[0;36mstart\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    861\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    862\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 863\u001b[1;33m                     \u001b[0mevent_pairs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_impl\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpoll\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpoll_timeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    864\u001b[0m                 \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    865\u001b[0m                     \u001b[1;31m# Depending on python version and IOLoop implementation,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\envs\\tf_gpu\\lib\\site-packages\\tornado\\platform\\select.py\u001b[0m in \u001b[0;36mpoll\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m     61\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpoll\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m         readable, writeable, errors = select.select(\n\u001b[1;32m---> 63\u001b[1;33m             self.read_fds, self.write_fds, self.error_fds, timeout)\n\u001b[0m\u001b[0;32m     64\u001b[0m         \u001b[0mevents\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     65\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mfd\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mreadable\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from gym_airsim.airsim_car_env import AirSimCarEnv\n",
    "import numpy as np\n",
    "from pathlib import Path\n",
    "import os\n",
    "from os.path import exists\n",
    "from stable_baselines.common.callbacks import CheckpointCallback, EvalCallback, CallbackList\n",
    "\n",
    "import tensorflow as tf\n",
    "from keras.backend.tensorflow_backend import set_session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.allow_growth = True  #dynamically grow the memory used on the GPU\n",
    "set_session(tf.Session(config=config))\n",
    "\n",
    "env = AirSimCarEnv()\n",
    "np.random.seed(123)\n",
    "\n",
    "from stable_baselines.common.policies import CnnPolicy\n",
    "from stable_baselines import PPO2\n",
    "\n",
    "log_dir = 'logs'\n",
    "if not exists(log_dir):\n",
    "    os.makedirs(log_dir)\n",
    "checkpoints_dir = Path(\"logs/ppo2_AirsimCar_checkpoints/\")\n",
    "tensorboard_dir = Path(\"logs/ppo2_AirsimCar_tensorboard/\")\n",
    "best_model_dir = Path(\"logs/ppo2_AirsimCar_best_model\")\n",
    "eval_dir = Path(\"logs/ppo2_AirsimCar_eval_results\")\n",
    "if not exists(checkpoints_dir):\n",
    "    os.makedirs(checkpoints_dir)\n",
    "if not exists(tensorboard_dir):\n",
    "    os.makedirs(tensorboard_dir)\n",
    "if not exists(best_model_dir):\n",
    "    os.makedirs(best_model_dir)\n",
    "if not exists(eval_dir):\n",
    "    os.makedirs(eval_dir)\n",
    "    \n",
    "\n",
    "checkpoint_callback = CheckpointCallback(save_freq=25000, save_path=checkpoints_dir,\n",
    "                                         name_prefix='ppo2_AirsimCar')\n",
    "eval_callback = EvalCallback(env, best_model_save_path=best_model_dir,\n",
    "                             log_path=eval_dir, eval_freq=25000)\n",
    "\n",
    "# Create the callback list\n",
    "callbacks = CallbackList([checkpoint_callback, eval_callback])\n",
    "\n",
    "model = PPO2(CnnPolicy, env, verbose=1, tensorboard_log=tensorboard_dir)\n",
    "model.learn(total_timesteps=2000000, callback=callbacks, log_interval=10)\n",
    "\n",
    "obs = env.reset()\n",
    "for i in range(1000):\n",
    "    action, _states = model.predict(obs)\n",
    "    obs, rewards, done, info = env.step(action)\n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}