main_code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning Agent Code (ME5418 Group 18)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Please edit the `TRAINING` and `Policy` Variable below, then run through every cell."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAINING = False # Set to True if you want to train from scratch. Set to False if you want to test our model.\n",
    "Policy = 3 # Set which policy you want to test/evaluate. For training, set to 0."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import libraries Cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 12698707614472741894\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import pickle\n",
    "import imageio\n",
    "\n",
    "# Import gym environment and DDPG (actor, critic, target) networks\n",
    "from aliengo_gym import AliengoEnv\n",
    "from DDPGNet import DDPGNet\n",
    "from DDPGNet import scale_actions\n",
    "from DDPGNet import scale_observation\n",
    "from DDPGNet import OUNoise\n",
    "\n",
    "# check your computer information\n",
    "from tensorflow.python.client import device_lib\n",
    "dev_list = device_lib.list_local_devices()\n",
    "print(dev_list) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameter Cell: Initialize parameters and helper functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Default parameters:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path to the folders saving the model parameters, render gifs, tf summary, and episode buffer informations\n",
    "model_path = os.path.dirname(os.path.abspath('__file__')) +  \"/model_save\"\n",
    "gifs_path = os.path.dirname(os.path.abspath('__file__')) + \"/gifs_save\"\n",
    "train_path = os.path.dirname(os.path.abspath('__file__')) + \"/tf_summary_save\"\n",
    "episode_buffer_path = os.path.dirname(os.path.abspath('__file__')) + \"/episode_buffer_save\"\n",
    "\n",
    "# To load previously trained model?\n",
    "\n",
    "if TRAINING == True:\n",
    "    Policy = 0 # Set to 0 if you want to start from scratch. Else, set your policy value.\n",
    "\n",
    "if Policy == 0:\n",
    "    load_model = False # Toggle True to load previously trained model\n",
    "elif Policy == 1:\n",
    "    load_model = True\n",
    "    MODEL_NUMBER = 2600 # which model number you want to load?\n",
    "elif Policy == 2:\n",
    "    load_model = True\n",
    "    MODEL_NUMBER = 500 # which model number you want to load?\n",
    "elif Policy == 3:\n",
    "    load_model = True\n",
    "    MODEL_NUMBER = 200 # which model number you want to load?\n",
    "else:\n",
    "    load_model = False\n",
    "    MODEL_NUMBER = 0\n",
    "\n",
    "# Parameters that decides when to save the model checkpoint, render gifs, tf summary and episode buffer informations.\n",
    "\n",
    "# Save GIF\n",
    "OUTPUT_GIFS = True # toggle True to save GIFS\n",
    "NEXT_GIF_COUNT = 100 # To save GIFS at every NEXT_GIFS_COUNT episodes interval\n",
    "\n",
    "# Save episode buffer, tf summary and model checkpoint\n",
    "SAVE_EPISODE_BUFFER = True # toggle True to save episode buffer\n",
    "SUMMARY_WINDOW = 5  # To save tf summary, and save model checkpoint at every SUMMARY_WINDOW episodes interval\n",
    "\n",
    "# global variables used tensorboard visualization. tf summary will be saved to these two variable and regularly  written out to local files\n",
    "episode_rewards = [] # do not edit\n",
    "episode_lengths = [] # do not edit\n",
    "\n",
    "# TRAINING and TESTING: \n",
    "MAX_EPISODE_LENGTH = 1000 # maximum steps to run in each episode during testing\n",
    "\n",
    "episode_count = 0 # (do not edit this variable) gloal variable to save how much episodes has already been gone through\n",
    "\n",
    "# TESTING\n",
    "NUM_EXPS = 1 # How many episodes you want to test?\n",
    "\n",
    "# TRAINING: for initializing actor, critic, target actor, target critic network\n",
    "observe_size = 40\n",
    "action_size = 8\n",
    "\n",
    "# TRAINING: for initializing actor, critic, target actor, target critic's trainer\n",
    "\n",
    "# TRAINING: for storing experience buffer and extracting a batch of experiences\n",
    "EXPERIENCE_BUFFER_SIZE = 10 # How much experience should be stored in the experience buffer before being used for backward pass.\n",
    "BATCH_SIZE = 10 # How much experiences should be extracted out from the experience buffer to be used for backward pass\n",
    "\n",
    "# TRAINING: for OU Noise parameter\n",
    "MU = 0.0\n",
    "THETA = 0.15\n",
    "SIGMA = 0.2\n",
    "DECAY_R = 0.99\n",
    "RESET_INTERVAL = 100\n",
    "\n",
    "# - for updating actor network\n",
    "ACTOR_LR = 1e-3\n",
    "ACTOR_L2_REG = 1e-5\n",
    "ACTOR_GRAD_CLIP = 1\n",
    "# - for updating critic network\n",
    "CRITIC_LR = 1e-3\n",
    "CRITIC_L2_REG = 2e-4\n",
    "CRITIC_GRAD_CLIP = 1\n",
    "# - for soft updating target network\n",
    "GAMMA = 0.99 \n",
    "TAU = 1e-3\n",
    "\n",
    "# Save render to gif helper function\n",
    "def make_gif(images, fname):\n",
    "    imageio.mimwrite(fname, images, subrectangles=True)\n",
    "    print(f\"Episode {episode_count - 1}. GIF is saved\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Worker Cell: To define Worker Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Worker:\n",
    "    \n",
    "    def __init__(self, game, global_network, noise):\n",
    "        self.env = game # Give it access to our GYM environment\n",
    "        self.global_network = global_network # Give it access to our actor, critic, target actor, target critic network\n",
    "        self.noise = noise # Give it access to our noise instance\n",
    "\n",
    "    def train(self, rollout, sess):\n",
    "        global episode_count\n",
    "        \n",
    "        # To randomly sample a few of experience from the replay buffer, then do some reshaping of their dimension to fit the dimension of our input layers.\n",
    "        batch = np.array(random.sample(rollout, min(BATCH_SIZE, len(rollout))), dtype=object) \n",
    "        current_observations = np.stack(batch[:, 0])\n",
    "        actions = np.stack(batch[:, 1])\n",
    "        next_observations = np.stack(batch[:, 2])\n",
    "        rewards = np.stack(batch[:, 3])\n",
    "\n",
    "        # Backward pass for actor network, obtain actor loss\n",
    "        feed_dict = {self.global_network.observation_input_placeholder: current_observations}\n",
    "\n",
    "        actor_loss, _ = sess.run([self.global_network.actor_loss, self.global_network.actor_trainer], feed_dict = feed_dict)\n",
    "\n",
    "        # Backward pass for critic network, obtain critic loss\n",
    "        feed_dict = {self.global_network.observation_input_placeholder: current_observations,\n",
    "                     self.global_network.action_input_placeholder: actions,\n",
    "                     self.global_network.reward_placeholder: rewards,\n",
    "                     self.global_network.next_observation_input_placeholder: next_observations}\n",
    "        \n",
    "        critic_loss, _ = sess.run([self.global_network.critic_loss, self.global_network.critic_trainer], feed_dict=feed_dict) \n",
    "\n",
    "        # Soft update for Target Actor and Target Critics\n",
    "        _ = sess.run(self.global_network.update_target_ops)\n",
    "\n",
    "        return actor_loss, critic_loss\n",
    "\n",
    "    def shouldRun(self, coord, episode_count): # Just a method to decide when to stop the training / testing operation\n",
    "        if TRAINING:\n",
    "            return (not coord.should_stop()) # For training, this is just an infinity loop\n",
    "        else:\n",
    "            return (episode_count < NUM_EXPS) # For testing, it will only run for NUM_EXPS episodes.\n",
    "\n",
    "    def work(self, sess, coord, saver, summary): \n",
    "\n",
    "        global episode_count, episode_rewards, episode_lengths # global variable for saving information that would be passed to global_summary at fixed interval\n",
    "        # episode_count -> how many episode has been gone through\n",
    "        # episode_rewards -> a list of (the sum of all reward obtain at each step at current episode)\n",
    "        # episode_lengths -> a list of (how many steps are taken at current episode)\n",
    "\n",
    "        self.global_summary = summary # our tf summary file writer -> for tensorboard visualization use\n",
    "\n",
    "        self.nextGIF = episode_count # nextGIF will decide at which episode should the training render be saved as GIF\n",
    "\n",
    "        with sess.as_default(), sess.graph.as_default(): # session (sess) and its associated graph become the default session and default graph.\n",
    "\n",
    "            while self.shouldRun(coord, episode_count): # LOOP THRU EPISODES\n",
    "\n",
    "                episode_buffer = [] # Clean out the episode buffer. Used to save our experiences\n",
    "                episode_reward = 0 # Clean out the episode reward. Used to save all \n",
    "                \n",
    "                d = False # Has it reached the termination state? Later on this value will be re-evaluated after each env.step()\n",
    "                saveGIF = False # Do I need to save GIF for this episode? Later on this value will be re-evaluated according to the nextGIF value.\n",
    "\n",
    "                self.env.reset(reload_urdf=False) # Reset environemnt -> return aliengo robot back to initial position orientation.\n",
    "                self.noise.reset() # reset OU noise state\n",
    "                \n",
    "                if OUTPUT_GIFS:\n",
    "\n",
    "                    if (not TRAINING) or (episode_count >= self.nextGIF): # To evaluate whether saving GIF is needed. Always True for testing. Only once a NEXT_GIF_COUNT for training\n",
    "                        saveGIF = True\n",
    "                        self.nextGIF = episode_count + NEXT_GIF_COUNT\n",
    "                        GIF_episode = int(episode_count)\n",
    "                        episode_frames = [self.env.render()] # save the render RGB array into our episode_frames\n",
    "                \n",
    "                episode_step_count = 0 # Number of steps taken in one episode\n",
    "\n",
    "                while True: # LOOP THRU STEPS\n",
    "\n",
    "                    # Please note that:\n",
    "                    # s: current observation. a: action taken. s1: next observation. r: reward. d: is it termination state?\n",
    "\n",
    "                    if episode_step_count == 0:\n",
    "                        s = self.env.get_state() # Get current state\n",
    "                        a = [0,0,0,0,0,0,0,0]\n",
    "\n",
    "                    # Just reshaping the state data\n",
    "                    s_list = []\n",
    "                    s_list.append((scale_observation(s, a), )) # normalize state values\n",
    "                    s_list = np.array(s_list, dtype=object)\n",
    "                    s = np.stack(s_list[:, 0])\n",
    "\n",
    "                    # Obtain the action as output by our actor network based on current state\n",
    "                    feed_dict = {self.global_network.observation_input_placeholder: s}\n",
    "                    a = sess.run(self.global_network.actor_net_output, feed_dict = feed_dict)\n",
    "\n",
    "                    if TRAINING:\n",
    "                        a = self.noise.get_action(action=a) # Add noise to our state using OU Noise\n",
    "                    \n",
    "                    a = (a + 1)*(26.5+26.5)/2 - 26.5 # undo the normalization\n",
    "                    a = tuple(a[0]) # reshaping the action value\n",
    "                    \n",
    "                    # Perform a step to obtain next state, reward, is it termination state\n",
    "                    s, _, s1, r, d, pa = self.env.step(a , episode_step_count)\n",
    "                    \n",
    "                    if saveGIF:\n",
    "                        episode_frames.append(self.env.render()) # only add a new RGB array of current render if saving GIF is required\n",
    "\n",
    "                    episode_buffer.append((scale_observation(s, scale_actions(pa)), scale_actions(a), scale_observation(s1, scale_actions(a)), np.array(r), np.array(d))) # then append the experience (s, a, s1, r, d) to our episode buffer\n",
    "                    episode_reward += r # add value to our episode_reward\n",
    "                    s = s1 # now next observation will be our current observation for the next loop\n",
    "                    \n",
    "                    episode_step_count +=1 # update episode step count\n",
    "\n",
    "                    if TRAINING and (len(episode_buffer) % EXPERIENCE_BUFFER_SIZE == 0 or d): # once it episode buffer has stored as much as the EXPERIENCE BUFFER SIZE, backward pass will be performed.\n",
    "                        if (len(episode_buffer) >= EXPERIENCE_BUFFER_SIZE):\n",
    "                            episode_buffer_training = episode_buffer[-EXPERIENCE_BUFFER_SIZE:] # Only obtain the last EXPERIENCE_BUFFER_SIZE of experiences for backward pass\n",
    "                        else:\n",
    "                            episode_buffer_training = episode_buffer[:] # Get all of the experience buffer to be used for training\n",
    "                        \n",
    "                        actor_loss, critic_loss = self.train(episode_buffer_training, sess) # calling the train method\n",
    "\n",
    "                        print(f\"Episode {episode_count}. Training Carried out at step = {episode_step_count}. Actorloss = {actor_loss}, Critic_loss = {critic_loss}\")\n",
    "\n",
    "                    if episode_step_count >= MAX_EPISODE_LENGTH or d: # check if termination state is reached \n",
    "                        if d == True:\n",
    "                            print(f\"Episode: {episode_count}. Termination state hit at step = {episode_step_count}\") \n",
    "                        break\n",
    "                \n",
    "                # Update our global variables used for tf.summary\n",
    "                episode_lengths.append(episode_step_count)\n",
    "                episode_rewards.append(episode_reward)\n",
    "\n",
    "                episode_count += 1\n",
    "\n",
    "                if not TRAINING: # Print our testing information\n",
    "                    print(f\"Episode {episode_count-1} Completed. Total step = {episode_step_count}. \\n\")\n",
    "                    print(\"Saving GIF ......\")\n",
    "                    GIF_episode = int(episode_count)\n",
    "                \n",
    "                else: # if TRAINING\n",
    "\n",
    "                    if episode_count % SUMMARY_WINDOW == 0: # check if needed to save tf summary for tensorboard visualization, and update our model checkpoint\n",
    "\n",
    "                        # Save Model Checkpoint\n",
    "\n",
    "                        saver.save(sess, model_path+'/model-'+str(int(episode_count))+'.ckpt')\n",
    "                        print(f\"Episode {episode_count-1}. Model checkpoint is saved \\n\")\n",
    "                        \n",
    "                        # To update our tf.summary and Tensorboard Visualization\n",
    "\n",
    "                        summary = tf.Summary()\n",
    "\n",
    "                        mean_reward = np.nanmean(episode_rewards[-SUMMARY_WINDOW:])\n",
    "                        mean_length = np.nanmean(episode_lengths[-SUMMARY_WINDOW:]) \n",
    "                        summary.value.add(tag='Perf/Reward', simple_value = mean_reward)\n",
    "                        summary.value.add(tag='Perf/Length', simple_value = mean_length) \n",
    "                        summary.value.add(tag=\"Losses/Actor Loss\", simple_value = actor_loss )\n",
    "                        summary.value.add(tag=\"Losses/Critic Loss\", simple_value = critic_loss )# In truth, within one episode, there will be multiple backward pass. Here only takes in the last backward pass loss\n",
    "\n",
    "                        self.global_summary.add_summary(summary, int(episode_count))\n",
    "                        self.global_summary.flush()\n",
    "\n",
    "                        print(f\"Episode {episode_count-1}. tf.summary -> Tensorboard is updated \\n\")\n",
    "\n",
    "                if saveGIF:\n",
    "\n",
    "                    # To generate GIF file. file name format: episode_<which episodes>_<how many steps taken>_<how much is the reward in this episode>\n",
    "\n",
    "                    images = np.array(episode_frames)\n",
    "                    if TRAINING:\n",
    "                        make_gif(images, \"{}/episode_{:d}_{:d}_{:.1f}.gif\".format(gifs_path, GIF_episode, episode_step_count, episode_reward))\n",
    "                    else:\n",
    "                        make_gif(images, '{}/episode_{:d}_{:d}.gif'.format(gifs_path, GIF_episode, episode_step_count))\n",
    "                \n",
    "                if SAVE_EPISODE_BUFFER:\n",
    "\n",
    "                    # To save the experience buffer. file name format: episode_<which episode>\n",
    "\n",
    "                    with open(episode_buffer_path + \"/episode_{}.dat\".format(GIF_episode), \"wb\") as file:\n",
    "                        pickle.dump(episode_buffer, file)\n",
    "\n",
    "        self.env.disconnect() # To terminate the PyBullet window after training is over. This would not run if training is stop manually, thus the extra last cell in this notebook.\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Main code Cell: to start training / testing\n",
    "Training normally would run in infinity loop. To terminate, just stop the cell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello World!\n",
      "Episode 0. Training Carried out at step = 10. Actorloss = -0.10084936767816544, Critic_loss = 0.19393768906593323\n",
      "Episode 0. Training Carried out at step = 20. Actorloss = -0.10078724473714828, Critic_loss = 0.23839206993579865\n",
      "Episode 0. Training Carried out at step = 30. Actorloss = -0.10066451877355576, Critic_loss = 0.08062668144702911\n",
      "Episode 0. Training Carried out at step = 40. Actorloss = -0.10048141330480576, Critic_loss = 0.7169147729873657\n",
      "Episode 0. Training Carried out at step = 50. Actorloss = -0.10023824870586395, Critic_loss = 2.960627317428589\n",
      "Episode 0. Training Carried out at step = 60. Actorloss = -0.09993439167737961, Critic_loss = 3.127807855606079\n",
      "Episode 0. Training Carried out at step = 70. Actorloss = -0.09956994652748108, Critic_loss = 0.15807391703128815\n",
      "Episode 0. Training Carried out at step = 80. Actorloss = -0.09914487600326538, Critic_loss = 0.08000321686267853\n",
      "Episode 0. Training Carried out at step = 90. Actorloss = -0.09867988526821136, Critic_loss = 0.11572252959012985\n",
      "Episode 0. Training Carried out at step = 100. Actorloss = -0.09819163382053375, Critic_loss = 0.0844176635146141\n",
      "Episode 0. Training Carried out at step = 110. Actorloss = -0.09769285470247269, Critic_loss = 0.08184611052274704\n",
      "Episode 0. Training Carried out at step = 120. Actorloss = -0.09719406068325043, Critic_loss = 0.061736371368169785\n",
      "Episode 0. Training Carried out at step = 130. Actorloss = -0.09670402854681015, Critic_loss = 0.005886937957257032\n",
      "Episode 0. Training Carried out at step = 140. Actorloss = -0.09622959047555923, Critic_loss = 0.0033108582720160484\n",
      "Episode 0. Training Carried out at step = 150. Actorloss = -0.09576620906591415, Critic_loss = 0.01769605278968811\n",
      "Episode 0. Training Carried out at step = 160. Actorloss = -0.09530431032180786, Critic_loss = 0.025503812357783318\n",
      "Episode 0. Training Carried out at step = 170. Actorloss = -0.09483558684587479, Critic_loss = 0.0077078272588551044\n",
      "Episode 0. Training Carried out at step = 180. Actorloss = -0.0943535640835762, Critic_loss = 0.0021409266628324986\n",
      "Episode 0. Training Carried out at step = 190. Actorloss = -0.09385696053504944, Critic_loss = 0.0021699476055800915\n",
      "Episode 0. Training Carried out at step = 200. Actorloss = -0.09334976226091385, Critic_loss = 0.00455308984965086\n",
      "Episode 0. Training Carried out at step = 210. Actorloss = -0.09283870458602905, Critic_loss = 0.005308548919856548\n",
      "Episode 0. Training Carried out at step = 220. Actorloss = -0.09233085811138153, Critic_loss = 0.0035421140491962433\n",
      "Episode 0. Training Carried out at step = 230. Actorloss = -0.09183119982481003, Critic_loss = 0.001980007393285632\n",
      "Episode 0. Training Carried out at step = 240. Actorloss = -0.09133661538362503, Critic_loss = 0.004679025616496801\n",
      "Episode 0. Training Carried out at step = 250. Actorloss = -0.09084095805883408, Critic_loss = 0.00482729310169816\n",
      "Episode 0. Training Carried out at step = 260. Actorloss = -0.09033864736557007, Critic_loss = 0.0030882800929248333\n",
      "Episode 0. Training Carried out at step = 270. Actorloss = -0.0898260772228241, Critic_loss = 0.0015125975478440523\n",
      "Episode 0. Training Carried out at step = 280. Actorloss = -0.08930275589227676, Critic_loss = 0.001880146563053131\n",
      "Episode 0. Training Carried out at step = 290. Actorloss = -0.08877205103635788, Critic_loss = 0.0037095099687576294\n",
      "Episode 0. Training Carried out at step = 300. Actorloss = -0.08823995292186737, Critic_loss = 0.00300364731810987\n",
      "Episode 0. Training Carried out at step = 310. Actorloss = -0.08771084994077682, Critic_loss = 0.002084777457639575\n",
      "Episode 0. Training Carried out at step = 320. Actorloss = -0.08718690276145935, Critic_loss = 0.0015611561248078942\n",
      "Episode 0. Training Carried out at step = 330. Actorloss = -0.08666582405567169, Critic_loss = 0.0019586177077144384\n",
      "Episode 0. Training Carried out at step = 340. Actorloss = -0.0861458107829094, Critic_loss = 0.003658315632492304\n",
      "Episode 0. Training Carried out at step = 350. Actorloss = -0.0856214091181755, Critic_loss = 0.0025831786915659904\n",
      "Episode 0. Training Carried out at step = 360. Actorloss = -0.08508898317813873, Critic_loss = 0.0013973511522635818\n",
      "Episode 0. Training Carried out at step = 370. Actorloss = -0.08454878628253937, Critic_loss = 0.001522503443993628\n",
      "Episode 0. Training Carried out at step = 380. Actorloss = -0.08400336652994156, Critic_loss = 0.003476787358522415\n",
      "Episode 0. Training Carried out at step = 390. Actorloss = -0.083456851541996, Critic_loss = 0.002588248811662197\n",
      "Episode 0. Training Carried out at step = 400. Actorloss = -0.08291418850421906, Critic_loss = 0.0018242739606648684\n",
      "Episode 0. Training Carried out at step = 410. Actorloss = -0.0823788121342659, Critic_loss = 0.0014471677131950855\n",
      "Episode 0. Training Carried out at step = 420. Actorloss = -0.0818479061126709, Critic_loss = 0.0027252240106463432\n",
      "Episode 0. Training Carried out at step = 430. Actorloss = -0.0813167542219162, Critic_loss = 0.003118238877505064\n",
      "Episode 0. Training Carried out at step = 440. Actorloss = -0.08078046143054962, Critic_loss = 0.001930820057168603\n",
      "Episode 0. Training Carried out at step = 450. Actorloss = -0.0802365392446518, Critic_loss = 0.001317050657235086\n",
      "Episode 0. Training Carried out at step = 460. Actorloss = -0.07968604564666748, Critic_loss = 0.002141315955668688\n",
      "Episode 0. Training Carried out at step = 470. Actorloss = -0.07913315296173096, Critic_loss = 0.002453475259244442\n",
      "Episode 0. Training Carried out at step = 480. Actorloss = -0.07858233153820038, Critic_loss = 0.0016603252151980996\n",
      "Episode 0. Training Carried out at step = 490. Actorloss = -0.07803580164909363, Critic_loss = 0.0013504931703209877\n",
      "Episode 0. Training Carried out at step = 500. Actorloss = -0.0774945542216301, Critic_loss = 0.0026623925659805536\n",
      "Episode 0. Training Carried out at step = 510. Actorloss = -0.07695044577121735, Critic_loss = 0.002799571957439184\n",
      "Episode 0. Training Carried out at step = 520. Actorloss = -0.07639934122562408, Critic_loss = 0.0016099964268505573\n",
      "Episode 0. Training Carried out at step = 530. Actorloss = -0.07584019750356674, Critic_loss = 0.0015305192209780216\n",
      "Episode 0. Training Carried out at step = 540. Actorloss = -0.07527609169483185, Critic_loss = 0.0023322475608438253\n",
      "Episode 0. Training Carried out at step = 550. Actorloss = -0.07471132278442383, Critic_loss = 0.00215919385664165\n",
      "Episode 0. Training Carried out at step = 560. Actorloss = -0.0741499811410904, Critic_loss = 0.0013985333498567343\n",
      "Episode 0. Training Carried out at step = 570. Actorloss = -0.07359305769205093, Critic_loss = 0.0014772856375202537\n",
      "Episode 0. Training Carried out at step = 580. Actorloss = -0.0730384811758995, Critic_loss = 0.002061054576188326\n",
      "Episode 0. Training Carried out at step = 590. Actorloss = -0.0724797248840332, Critic_loss = 0.0013134570326656103\n",
      "Episode 0. Training Carried out at step = 600. Actorloss = -0.07192160934209824, Critic_loss = 0.0013652232009917498\n",
      "Episode 0. Training Carried out at step = 610. Actorloss = -0.07136421650648117, Critic_loss = 0.0012951220851391554\n",
      "Episode 0. Training Carried out at step = 620. Actorloss = -0.07080593705177307, Critic_loss = 0.0013231325428932905\n",
      "Episode 0. Training Carried out at step = 630. Actorloss = -0.07024849206209183, Critic_loss = 0.0013212980702519417\n",
      "Episode 0. Training Carried out at step = 640. Actorloss = -0.06969009339809418, Critic_loss = 0.0013280108105391264\n",
      "Episode 0. Training Carried out at step = 650. Actorloss = -0.06913008540868759, Critic_loss = 0.0013029410038143396\n",
      "Episode 0. Training Carried out at step = 660. Actorloss = -0.06856777518987656, Critic_loss = 0.0013357202988117933\n",
      "Episode 0. Training Carried out at step = 670. Actorloss = -0.0680040642619133, Critic_loss = 0.0015103318728506565\n",
      "Episode 0. Training Carried out at step = 680. Actorloss = -0.06744126230478287, Critic_loss = 0.001466400921344757\n",
      "Episode 0. Training Carried out at step = 690. Actorloss = -0.06688309460878372, Critic_loss = 0.0013290323549881577\n",
      "Episode 0. Training Carried out at step = 700. Actorloss = -0.06631531566381454, Critic_loss = 0.0019284895388409495\n",
      "Episode 0. Training Carried out at step = 710. Actorloss = -0.06576019525527954, Critic_loss = 0.0013090278953313828\n",
      "Episode 0. Training Carried out at step = 720. Actorloss = -0.06521286815404892, Critic_loss = 0.0017183037707582116\n",
      "Episode 0. Training Carried out at step = 730. Actorloss = -0.06466555595397949, Critic_loss = 0.0014226693892851472\n",
      "Episode 0. Training Carried out at step = 740. Actorloss = -0.06411974132061005, Critic_loss = 0.0021779686212539673\n",
      "Episode 0. Training Carried out at step = 750. Actorloss = -0.06357603520154953, Critic_loss = 0.0015583067433908582\n",
      "Episode 0. Training Carried out at step = 760. Actorloss = -0.06303733587265015, Critic_loss = 0.00410066545009613\n",
      "Episode 0. Training Carried out at step = 770. Actorloss = -0.06249423325061798, Critic_loss = 0.0017316814046353102\n",
      "Episode 0. Training Carried out at step = 780. Actorloss = -0.061942748725414276, Critic_loss = 0.055478278547525406\n",
      "Episode 0. Training Carried out at step = 790. Actorloss = -0.06139635294675827, Critic_loss = 0.18389038741588593\n",
      "Episode 0. Training Carried out at step = 800. Actorloss = -0.060869213193655014, Critic_loss = 0.04940744861960411\n",
      "Episode 0. Training Carried out at step = 810. Actorloss = -0.06038513407111168, Critic_loss = 0.005729591473937035\n",
      "Episode 0. Training Carried out at step = 820. Actorloss = -0.059931136667728424, Critic_loss = 0.0031342750880867243\n",
      "Episode 0. Training Carried out at step = 830. Actorloss = -0.05949293449521065, Critic_loss = 0.0029211589135229588\n",
      "Episode 0. Training Carried out at step = 840. Actorloss = -0.0590866394340992, Critic_loss = 0.0030979844741523266\n",
      "Episode 0. Training Carried out at step = 850. Actorloss = -0.05870512127876282, Critic_loss = 0.003024537581950426\n",
      "Episode 0. Training Carried out at step = 860. Actorloss = -0.05834963917732239, Critic_loss = 0.0034950473345816135\n",
      "Episode 0. Training Carried out at step = 870. Actorloss = -0.05802477151155472, Critic_loss = 0.062198854982852936\n",
      "Episode 0. Training Carried out at step = 880. Actorloss = -0.05771690234541893, Critic_loss = 0.04647376388311386\n",
      "Episode 0. Training Carried out at step = 890. Actorloss = -0.05739891529083252, Critic_loss = 0.02960759773850441\n",
      "Episode 0. Training Carried out at step = 900. Actorloss = -0.0570787712931633, Critic_loss = 0.012668367475271225\n",
      "Episode 0. Training Carried out at step = 910. Actorloss = -0.05674832686781883, Critic_loss = 0.11505032330751419\n",
      "Episode 0. Training Carried out at step = 920. Actorloss = -0.05641859769821167, Critic_loss = 0.2413313090801239\n",
      "Episode 0. Training Carried out at step = 930. Actorloss = -0.056112032383680344, Critic_loss = 0.28555458784103394\n",
      "Episode 0. Training Carried out at step = 940. Actorloss = -0.05582096055150032, Critic_loss = 0.42708826065063477\n",
      "Episode 0. Training Carried out at step = 950. Actorloss = -0.05555785447359085, Critic_loss = 0.1023569107055664\n",
      "Episode 0. Training Carried out at step = 960. Actorloss = -0.055308081209659576, Critic_loss = 0.06522312760353088\n",
      "Episode 0. Training Carried out at step = 970. Actorloss = -0.055095650255680084, Critic_loss = 0.6491129994392395\n",
      "Episode 0. Training Carried out at step = 980. Actorloss = -0.054887764155864716, Critic_loss = 1.1934926509857178\n",
      "Episode 0. Training Carried out at step = 990. Actorloss = -0.05467955395579338, Critic_loss = 0.8834441900253296\n",
      "Episode 0. Training Carried out at step = 1000. Actorloss = -0.05446070432662964, Critic_loss = 0.8472828269004822\n",
      "Episode 0. GIF is saved\n",
      "Episode 1. Training Carried out at step = 10. Actorloss = -0.05405634269118309, Critic_loss = 4.991430282592773\n",
      "Episode 1. Training Carried out at step = 20. Actorloss = -0.05379077419638634, Critic_loss = 19.05099868774414\n",
      "Episode 1. Training Carried out at step = 30. Actorloss = -0.05348649621009827, Critic_loss = 24.085840225219727\n",
      "Episode 1. Training Carried out at step = 40. Actorloss = -0.053184330463409424, Critic_loss = 4.100254535675049\n",
      "Episode 1. Training Carried out at step = 50. Actorloss = -0.05281303450465202, Critic_loss = 0.15853826701641083\n",
      "Episode 1. Training Carried out at step = 60. Actorloss = -0.05240121856331825, Critic_loss = 0.21578563749790192\n",
      "Episode 1. Training Carried out at step = 70. Actorloss = -0.051972854882478714, Critic_loss = 0.2177990972995758\n",
      "Episode 1. Training Carried out at step = 80. Actorloss = -0.05153913423418999, Critic_loss = 0.17326200008392334\n",
      "Episode 1. Training Carried out at step = 90. Actorloss = -0.05111470818519592, Critic_loss = 0.12471769005060196\n",
      "Episode 1. Training Carried out at step = 100. Actorloss = -0.050702113658189774, Critic_loss = 0.1115729957818985\n",
      "Episode 1. Training Carried out at step = 110. Actorloss = -0.05031297355890274, Critic_loss = 0.09324856102466583\n",
      "Episode 1. Training Carried out at step = 120. Actorloss = -0.049954675137996674, Critic_loss = 0.0724918395280838\n",
      "Episode 1. Training Carried out at step = 130. Actorloss = -0.04963361471891403, Critic_loss = 0.05241658538579941\n",
      "Episode 1. Training Carried out at step = 140. Actorloss = -0.049363717436790466, Critic_loss = 0.03485960140824318\n",
      "Episode 1. Training Carried out at step = 150. Actorloss = -0.04914834350347519, Critic_loss = 0.0209856815636158\n",
      "Episode 1. Training Carried out at step = 160. Actorloss = -0.04896204546093941, Critic_loss = 0.01141146756708622\n",
      "Episode 1. Training Carried out at step = 170. Actorloss = -0.048808854073286057, Critic_loss = 0.006054960191249847\n",
      "Episode 1. Training Carried out at step = 180. Actorloss = -0.04869508743286133, Critic_loss = 0.0034716748632490635\n",
      "Episode 1. Training Carried out at step = 190. Actorloss = -0.04864306375384331, Critic_loss = 0.0015190653502941132\n",
      "Episode 1. Training Carried out at step = 200. Actorloss = -0.048621226102113724, Critic_loss = 0.09009160846471786\n",
      "Episode 1. Training Carried out at step = 210. Actorloss = -0.04862825945019722, Critic_loss = 0.23230278491973877\n",
      "Episode 1. Training Carried out at step = 220. Actorloss = -0.04865401238203049, Critic_loss = 0.34996524453163147\n",
      "Episode 1. Training Carried out at step = 230. Actorloss = -0.04869508370757103, Critic_loss = 0.431083619594574\n",
      "Episode 1. Training Carried out at step = 240. Actorloss = -0.048732396215200424, Critic_loss = 0.4280276596546173\n",
      "Episode 1. Training Carried out at step = 250. Actorloss = -0.04876699671149254, Critic_loss = 0.19571448862552643\n",
      "Episode 1. Training Carried out at step = 260. Actorloss = -0.04878031834959984, Critic_loss = 0.17717842757701874\n",
      "Episode 1. Training Carried out at step = 270. Actorloss = -0.048777833580970764, Critic_loss = 0.12885834276676178\n",
      "Episode 1. Training Carried out at step = 280. Actorloss = -0.04877222701907158, Critic_loss = 0.5837669372558594\n",
      "Episode 1. Training Carried out at step = 290. Actorloss = -0.04872209578752518, Critic_loss = 1.5102967023849487\n",
      "Episode 1. Training Carried out at step = 300. Actorloss = -0.04864000901579857, Critic_loss = 1.9767022132873535\n",
      "Episode 1. Training Carried out at step = 310. Actorloss = -0.04852619394659996, Critic_loss = 2.209768533706665\n",
      "Episode 1. Training Carried out at step = 320. Actorloss = -0.04837661609053612, Critic_loss = 2.0706727504730225\n",
      "Episode 1. Training Carried out at step = 330. Actorloss = -0.04818800464272499, Critic_loss = 1.8945828676223755\n",
      "Episode 1. Training Carried out at step = 340. Actorloss = -0.04795821011066437, Critic_loss = 1.2392008304595947\n",
      "Episode 1. Training Carried out at step = 350. Actorloss = -0.047684624791145325, Critic_loss = 0.5109907388687134\n",
      "Episode 1. Training Carried out at step = 360. Actorloss = -0.04736201465129852, Critic_loss = 0.10316193848848343\n",
      "Episode 1. Training Carried out at step = 370. Actorloss = -0.047000981867313385, Critic_loss = 0.1583285927772522\n",
      "Episode 1. Training Carried out at step = 380. Actorloss = -0.04661225900053978, Critic_loss = 0.16853518784046173\n",
      "Episode 1. Training Carried out at step = 390. Actorloss = -0.04620593413710594, Critic_loss = 0.16344527900218964\n",
      "Episode 1. Training Carried out at step = 400. Actorloss = -0.04579372704029083, Critic_loss = 0.10594899207353592\n",
      "Episode 1. Training Carried out at step = 410. Actorloss = -0.04538111016154289, Critic_loss = 0.09290013462305069\n",
      "Episode 1. Training Carried out at step = 420. Actorloss = -0.04497572034597397, Critic_loss = 0.08458247035741806\n",
      "Episode 1. Training Carried out at step = 430. Actorloss = -0.044584810733795166, Critic_loss = 0.07197441160678864\n",
      "Episode 1. Training Carried out at step = 440. Actorloss = -0.044214554131031036, Critic_loss = 0.05715997889637947\n",
      "Episode 1. Training Carried out at step = 450. Actorloss = -0.04386930912733078, Critic_loss = 0.04262472689151764\n",
      "Episode 1. Training Carried out at step = 460. Actorloss = -0.04355340823531151, Critic_loss = 0.02954808808863163\n",
      "Episode 1. Training Carried out at step = 470. Actorloss = -0.043268099427223206, Critic_loss = 0.01910228654742241\n",
      "Episode 1. Training Carried out at step = 480. Actorloss = -0.04301104694604874, Critic_loss = 0.011140406131744385\n",
      "Episode 1. Training Carried out at step = 490. Actorloss = -0.04276641085743904, Critic_loss = 0.005774879828095436\n",
      "Episode 1. Training Carried out at step = 500. Actorloss = -0.042572591453790665, Critic_loss = 0.0030382629483938217\n",
      "Episode 1. Training Carried out at step = 510. Actorloss = -0.04239724949002266, Critic_loss = 0.014090080745518208\n",
      "Episode 1. Training Carried out at step = 520. Actorloss = -0.04224942624568939, Critic_loss = 0.14228996634483337\n",
      "Episode 1. Training Carried out at step = 530. Actorloss = -0.042118143290281296, Critic_loss = 0.25258708000183105\n",
      "Episode 1. Training Carried out at step = 540. Actorloss = -0.04199383407831192, Critic_loss = 0.360339879989624\n",
      "Episode 1. Training Carried out at step = 550. Actorloss = -0.04187667369842529, Critic_loss = 0.4086686670780182\n",
      "Episode 1. Training Carried out at step = 560. Actorloss = -0.041764721274375916, Critic_loss = 0.22106678783893585\n",
      "Episode 1. Training Carried out at step = 570. Actorloss = -0.04164432734251022, Critic_loss = 0.16003701090812683\n",
      "Episode 1. Training Carried out at step = 580. Actorloss = -0.041512541472911835, Critic_loss = 0.13953489065170288\n",
      "Episode 1. Training Carried out at step = 590. Actorloss = -0.04137447103857994, Critic_loss = 0.11445636302232742\n",
      "Episode 1. Training Carried out at step = 600. Actorloss = -0.041227079927921295, Critic_loss = 1.1211743354797363\n",
      "Episode 1. Training Carried out at step = 610. Actorloss = -0.041054245084524155, Critic_loss = 1.6513670682907104\n",
      "Episode 1. Training Carried out at step = 620. Actorloss = -0.04086165130138397, Critic_loss = 1.9289989471435547\n",
      "Episode 1. Training Carried out at step = 630. Actorloss = -0.04064597934484482, Critic_loss = 1.9210139513015747\n",
      "Episode 1. Training Carried out at step = 640. Actorloss = -0.040406472980976105, Critic_loss = 1.8120242357254028\n",
      "Episode 1. Training Carried out at step = 650. Actorloss = -0.04013859108090401, Critic_loss = 1.3346259593963623\n",
      "Episode 1. Training Carried out at step = 660. Actorloss = -0.0398402214050293, Critic_loss = 0.5905342698097229\n",
      "Episode 1. Training Carried out at step = 670. Actorloss = -0.03950817510485649, Critic_loss = 0.10289277881383896\n",
      "Episode 1. Training Carried out at step = 680. Actorloss = -0.03914443030953407, Critic_loss = 0.23645251989364624\n",
      "Episode 1. Training Carried out at step = 690. Actorloss = -0.038756873458623886, Critic_loss = 0.2819470465183258\n",
      "Episode 1. Training Carried out at step = 700. Actorloss = -0.03835322707891464, Critic_loss = 0.2936505973339081\n",
      "Episode 1. Training Carried out at step = 710. Actorloss = -0.03794081136584282, Critic_loss = 0.27011969685554504\n",
      "Episode 1. Training Carried out at step = 720. Actorloss = -0.03752715140581131, Critic_loss = 0.2035171538591385\n",
      "Episode 1. Training Carried out at step = 730. Actorloss = -0.03711596131324768, Critic_loss = 0.20054449141025543\n",
      "Episode 1. Training Carried out at step = 740. Actorloss = -0.036713212728500366, Critic_loss = 0.18882137537002563\n",
      "Episode 1. Training Carried out at step = 750. Actorloss = -0.03632223978638649, Critic_loss = 0.17120051383972168\n",
      "Episode 1. Training Carried out at step = 760. Actorloss = -0.035949964076280594, Critic_loss = 0.14973387122154236\n",
      "Episode 1. Training Carried out at step = 770. Actorloss = -0.035596150904893875, Critic_loss = 0.12549203634262085\n",
      "Episode 1. Training Carried out at step = 780. Actorloss = -0.03526844084262848, Critic_loss = 0.10161353647708893\n",
      "Episode 1. Training Carried out at step = 790. Actorloss = -0.03496677801012993, Critic_loss = 0.0785149410367012\n",
      "Episode 1. Training Carried out at step = 800. Actorloss = -0.03468654304742813, Critic_loss = 0.053117815405130386\n",
      "Episode 1. Training Carried out at step = 810. Actorloss = -0.03444463759660721, Critic_loss = 0.04163351655006409\n",
      "Episode 1. Training Carried out at step = 820. Actorloss = -0.034235697239637375, Critic_loss = 0.01468874141573906\n",
      "Episode 1. Training Carried out at step = 830. Actorloss = -0.03405901417136192, Critic_loss = 0.0269640926271677\n",
      "Episode 1. Training Carried out at step = 840. Actorloss = -0.03390822187066078, Critic_loss = 0.1130516454577446\n",
      "Episode 1. Training Carried out at step = 850. Actorloss = -0.033775899559259415, Critic_loss = 0.22513175010681152\n",
      "Episode 1. Training Carried out at step = 860. Actorloss = -0.033655598759651184, Critic_loss = 0.31011199951171875\n",
      "Episode 1. Training Carried out at step = 870. Actorloss = -0.033543236553668976, Critic_loss = 0.23220917582511902\n",
      "Episode 1. Training Carried out at step = 880. Actorloss = -0.03343164548277855, Critic_loss = 0.12738007307052612\n",
      "Episode 1. Training Carried out at step = 890. Actorloss = -0.03331361711025238, Critic_loss = 0.11356943845748901\n",
      "Episode 1. Training Carried out at step = 900. Actorloss = -0.033187542110681534, Critic_loss = 0.10054735094308853\n",
      "Episode 1. Training Carried out at step = 910. Actorloss = -0.03305043280124664, Critic_loss = 0.30588847398757935\n",
      "Episode 1. Training Carried out at step = 920. Actorloss = -0.03289860486984253, Critic_loss = 1.483187198638916\n",
      "Episode 1. Training Carried out at step = 930. Actorloss = -0.03272779658436775, Critic_loss = 1.6874775886535645\n",
      "Episode 1. Training Carried out at step = 940. Actorloss = -0.032537028193473816, Critic_loss = 1.8900501728057861\n",
      "Episode 1. Training Carried out at step = 950. Actorloss = -0.03232334181666374, Critic_loss = 1.9284089803695679\n",
      "Episode 1. Training Carried out at step = 960. Actorloss = -0.032084617763757706, Critic_loss = 1.600671410560608\n",
      "Episode 1. Training Carried out at step = 970. Actorloss = -0.031818874180316925, Critic_loss = 0.8970890641212463\n",
      "Episode 1. Training Carried out at step = 980. Actorloss = -0.03152430057525635, Critic_loss = 0.21494697034358978\n",
      "Episode 1. Training Carried out at step = 990. Actorloss = -0.03119935467839241, Critic_loss = 0.1534474641084671\n",
      "Episode 1. Training Carried out at step = 1000. Actorloss = -0.0308515764772892, Critic_loss = 0.23821842670440674\n",
      "Episode 2. Training Carried out at step = 10. Actorloss = -0.030439430847764015, Critic_loss = 4.267268657684326\n",
      "Episode 2. Training Carried out at step = 20. Actorloss = -0.03008776344358921, Critic_loss = 17.214326858520508\n",
      "Episode 2. Training Carried out at step = 30. Actorloss = -0.029702845960855484, Critic_loss = 21.302522659301758\n",
      "Episode 2. Training Carried out at step = 40. Actorloss = -0.02927481383085251, Critic_loss = 2.7343358993530273\n",
      "Episode 2. Training Carried out at step = 50. Actorloss = -0.02882532775402069, Critic_loss = 0.3795457184314728\n",
      "Episode 2. Training Carried out at step = 60. Actorloss = -0.028362177312374115, Critic_loss = 0.4889514446258545\n",
      "Episode 2. Training Carried out at step = 70. Actorloss = -0.02788666822016239, Critic_loss = 0.47006940841674805\n",
      "Episode 2. Training Carried out at step = 80. Actorloss = -0.027403779327869415, Critic_loss = 0.4072837233543396\n",
      "Episode 2. Training Carried out at step = 90. Actorloss = -0.026918793097138405, Critic_loss = 0.34358787536621094\n",
      "Episode 2. Training Carried out at step = 100. Actorloss = -0.026437997817993164, Critic_loss = 0.3314197063446045\n",
      "Episode 2. Training Carried out at step = 110. Actorloss = -0.025965122506022453, Critic_loss = 0.30916649103164673\n",
      "Episode 2. Training Carried out at step = 120. Actorloss = -0.02550417184829712, Critic_loss = 0.28104886412620544\n",
      "Episode 2. Training Carried out at step = 130. Actorloss = -0.02505875565111637, Critic_loss = 0.24775998294353485\n",
      "Episode 2. Training Carried out at step = 140. Actorloss = -0.024629000574350357, Critic_loss = 0.21180182695388794\n",
      "Episode 2. Training Carried out at step = 150. Actorloss = -0.02421913854777813, Critic_loss = 0.17661358416080475\n",
      "Episode 2. Training Carried out at step = 160. Actorloss = -0.023839063942432404, Critic_loss = 0.1421840935945511\n",
      "Episode 2. Training Carried out at step = 170. Actorloss = -0.023487573489546776, Critic_loss = 0.11228752881288528\n",
      "Episode 2. Training Carried out at step = 180. Actorloss = -0.023172670975327492, Critic_loss = 0.09390714764595032\n",
      "Episode 2. Training Carried out at step = 190. Actorloss = -0.022877205163240433, Critic_loss = 0.06097203493118286\n",
      "Episode 2. Training Carried out at step = 200. Actorloss = -0.022608593106269836, Critic_loss = 0.023156702518463135\n",
      "Episode 2. Training Carried out at step = 210. Actorloss = -0.022369317710399628, Critic_loss = 0.08796804398298264\n",
      "Episode 2. Training Carried out at step = 220. Actorloss = -0.022152438759803772, Critic_loss = 0.17339472472667694\n",
      "Episode 2. Training Carried out at step = 230. Actorloss = -0.0219484344124794, Critic_loss = 0.2399182766675949\n",
      "Episode 2. Training Carried out at step = 240. Actorloss = -0.021753523498773575, Critic_loss = 0.26618528366088867\n",
      "Episode 2. Training Carried out at step = 250. Actorloss = -0.021553512662649155, Critic_loss = 0.09997093677520752\n",
      "Episode 2. Training Carried out at step = 260. Actorloss = -0.021359380334615707, Critic_loss = 0.0797189399600029\n",
      "Episode 2. Training Carried out at step = 270. Actorloss = -0.021164555102586746, Critic_loss = 0.04555850476026535\n",
      "Episode 2. Training Carried out at step = 280. Actorloss = -0.02095712162554264, Critic_loss = 0.5577378869056702\n",
      "Episode 2. Training Carried out at step = 290. Actorloss = -0.02074289135634899, Critic_loss = 1.0271739959716797\n",
      "Episode 2. Training Carried out at step = 300. Actorloss = -0.020517461001873016, Critic_loss = 1.4056875705718994\n",
      "Episode 2. Training Carried out at step = 310. Actorloss = -0.020277515053749084, Critic_loss = 1.5824925899505615\n",
      "Episode 2. Training Carried out at step = 320. Actorloss = -0.020017312839627266, Critic_loss = 1.571724772453308\n",
      "Episode 2. Training Carried out at step = 330. Actorloss = -0.019738059490919113, Critic_loss = 1.4442778825759888\n",
      "Episode 2. Training Carried out at step = 340. Actorloss = -0.019436825066804886, Critic_loss = 0.9699320197105408\n",
      "Episode 2. Training Carried out at step = 350. Actorloss = -0.019111961126327515, Critic_loss = 0.39394500851631165\n",
      "Episode 2. Training Carried out at step = 360. Actorloss = -0.018762236461043358, Critic_loss = 0.07736397534608841\n",
      "Episode 2. Training Carried out at step = 370. Actorloss = -0.018387969583272934, Critic_loss = 0.27948734164237976\n",
      "Episode 2. Training Carried out at step = 380. Actorloss = -0.017993489280343056, Critic_loss = 0.3009185791015625\n",
      "Episode 2. Training Carried out at step = 390. Actorloss = -0.017587192356586456, Critic_loss = 0.2982020974159241\n",
      "Episode 2. Training Carried out at step = 400. Actorloss = -0.017169909551739693, Critic_loss = 0.22382980585098267\n",
      "Episode 2. Training Carried out at step = 410. Actorloss = -0.016754621639847755, Critic_loss = 0.20874053239822388\n",
      "Episode 2. Training Carried out at step = 420. Actorloss = -0.01634632609784603, Critic_loss = 0.203809916973114\n",
      "Episode 2. Training Carried out at step = 430. Actorloss = -0.01594211347401142, Critic_loss = 0.19219204783439636\n",
      "Episode 2. Training Carried out at step = 440. Actorloss = -0.01554886344820261, Critic_loss = 0.17572087049484253\n",
      "Episode 2. Training Carried out at step = 450. Actorloss = -0.015173635445535183, Critic_loss = 0.1560835838317871\n",
      "Episode 2. Training Carried out at step = 460. Actorloss = -0.014813479036092758, Critic_loss = 0.13462451100349426\n",
      "Episode 2. Training Carried out at step = 470. Actorloss = -0.014467630535364151, Critic_loss = 0.11246126890182495\n",
      "Episode 2. Training Carried out at step = 480. Actorloss = -0.014149303548038006, Critic_loss = 0.0915449783205986\n",
      "Episode 2. Training Carried out at step = 490. Actorloss = -0.013878165744245052, Critic_loss = 0.07940541952848434\n",
      "Episode 2. Training Carried out at step = 500. Actorloss = -0.013617155142128468, Critic_loss = 0.05600745975971222\n",
      "Episode 2. Training Carried out at step = 510. Actorloss = -0.013375673443078995, Critic_loss = 0.011756297200918198\n",
      "Episode 2. Training Carried out at step = 520. Actorloss = -0.013144366443157196, Critic_loss = 0.04986371099948883\n",
      "Episode 2. Training Carried out at step = 530. Actorloss = -0.012946958653628826, Critic_loss = 0.13185028731822968\n",
      "Episode 2. Training Carried out at step = 540. Actorloss = -0.012764687649905682, Critic_loss = 0.22087591886520386\n",
      "Episode 2. Training Carried out at step = 550. Actorloss = -0.01258936058729887, Critic_loss = 0.2705900967121124\n",
      "Episode 2. Training Carried out at step = 560. Actorloss = -0.01239602081477642, Critic_loss = 0.1713353842496872\n",
      "Episode 2. Training Carried out at step = 570. Actorloss = -0.012208045460283756, Critic_loss = 0.09327979385852814\n",
      "Episode 2. Training Carried out at step = 580. Actorloss = -0.012031364254653454, Critic_loss = 0.08108088374137878\n",
      "Episode 2. Training Carried out at step = 590. Actorloss = -0.011854875832796097, Critic_loss = 0.14183054864406586\n",
      "Episode 2. Training Carried out at step = 600. Actorloss = -0.01165735349059105, Critic_loss = 0.8561082482337952\n",
      "Episode 2. Training Carried out at step = 610. Actorloss = -0.0114470599219203, Critic_loss = 1.2600184679031372\n",
      "Episode 2. Training Carried out at step = 620. Actorloss = -0.011228117160499096, Critic_loss = 1.7023144960403442\n",
      "Episode 2. Training Carried out at step = 630. Actorloss = -0.010998127982020378, Critic_loss = 1.7020593881607056\n",
      "Episode 2. Training Carried out at step = 640. Actorloss = -0.010741900652647018, Critic_loss = 1.6688804626464844\n",
      "Episode 2. Training Carried out at step = 650. Actorloss = -0.010464752092957497, Critic_loss = 1.313777208328247\n",
      "Episode 2. Training Carried out at step = 660. Actorloss = -0.01016779150813818, Critic_loss = 0.6513961553573608\n",
      "Episode 2. Training Carried out at step = 670. Actorloss = -0.009848532266914845, Critic_loss = 0.14764364063739777\n",
      "Episode 2. Training Carried out at step = 680. Actorloss = -0.009504501707851887, Critic_loss = 0.18236470222473145\n",
      "Episode 2. Training Carried out at step = 690. Actorloss = -0.009137837216258049, Critic_loss = 0.24334612488746643\n",
      "Episode 2. Training Carried out at step = 700. Actorloss = -0.008755771443247795, Critic_loss = 0.24571645259857178\n",
      "Episode 2. Training Carried out at step = 710. Actorloss = -0.008361738175153732, Critic_loss = 0.21228455007076263\n",
      "Episode 2. Training Carried out at step = 720. Actorloss = -0.00796480942517519, Critic_loss = 0.1576177477836609\n",
      "Episode 2. Training Carried out at step = 730. Actorloss = -0.007570380344986916, Critic_loss = 0.1555698961019516\n",
      "Episode 2. Training Carried out at step = 740. Actorloss = -0.0071799252182245255, Critic_loss = 0.14863604307174683\n",
      "Episode 2. Training Carried out at step = 750. Actorloss = -0.006803990341722965, Critic_loss = 0.1371893286705017\n",
      "Episode 2. Training Carried out at step = 760. Actorloss = -0.006435728166252375, Critic_loss = 0.12246035039424896\n",
      "Episode 2. Training Carried out at step = 770. Actorloss = -0.006092225667089224, Critic_loss = 0.10626401007175446\n",
      "Episode 2. Training Carried out at step = 780. Actorloss = -0.0057536582462489605, Critic_loss = 0.08959943056106567\n",
      "Episode 2. Training Carried out at step = 790. Actorloss = -0.005437877494841814, Critic_loss = 0.07290574163198471\n",
      "Episode 2. Training Carried out at step = 800. Actorloss = -0.005179019644856453, Critic_loss = 0.05688056722283363\n",
      "Episode 2. Training Carried out at step = 810. Actorloss = -0.004927553702145815, Critic_loss = 0.04658907651901245\n",
      "Episode 2. Training Carried out at step = 820. Actorloss = -0.004686913453042507, Critic_loss = 0.021527348086237907\n",
      "Episode 2. Training Carried out at step = 830. Actorloss = -0.004452966619282961, Critic_loss = 0.021434959024190903\n",
      "Episode 2. Training Carried out at step = 840. Actorloss = -0.004233434330672026, Critic_loss = 0.10024528205394745\n",
      "Episode 2. Training Carried out at step = 850. Actorloss = -0.00404506828635931, Critic_loss = 0.20708408951759338\n",
      "Episode 2. Training Carried out at step = 860. Actorloss = -0.003869426902383566, Critic_loss = 0.28369131684303284\n",
      "Episode 2. Training Carried out at step = 870. Actorloss = -0.003674435196444392, Critic_loss = 0.2486657202243805\n",
      "Episode 2. Training Carried out at step = 880. Actorloss = -0.0034579329658299685, Critic_loss = 0.11497178673744202\n",
      "Episode 2. Training Carried out at step = 890. Actorloss = -0.0032788689713925123, Critic_loss = 0.10156357288360596\n",
      "Episode 2. Training Carried out at step = 900. Actorloss = -0.0030993525870144367, Critic_loss = 0.08109847456216812\n",
      "Episode 2. Training Carried out at step = 910. Actorloss = -0.0029148058965802193, Critic_loss = 0.39490756392478943\n",
      "Episode 2. Training Carried out at step = 920. Actorloss = -0.0027014834340661764, Critic_loss = 1.2630192041397095\n",
      "Episode 2. Training Carried out at step = 930. Actorloss = -0.0024908140767365694, Critic_loss = 1.6132419109344482\n",
      "Episode 2. Training Carried out at step = 940. Actorloss = -0.002272629179060459, Critic_loss = 1.8364510536193848\n",
      "Episode 2. Training Carried out at step = 950. Actorloss = -0.002021652879193425, Critic_loss = 1.882411241531372\n",
      "Episode 2. Training Carried out at step = 960. Actorloss = -0.001757191144861281, Critic_loss = 1.6133228540420532\n",
      "Episode 2. Training Carried out at step = 970. Actorloss = -0.0014737220481038094, Critic_loss = 0.9799559116363525\n",
      "Episode 2. Training Carried out at step = 980. Actorloss = -0.001171040814369917, Critic_loss = 0.31807222962379456\n",
      "Episode 2. Training Carried out at step = 990. Actorloss = -0.0008602096349932253, Critic_loss = 0.09747928380966187\n",
      "Episode 2. Training Carried out at step = 1000. Actorloss = -0.0005090349586680532, Critic_loss = 0.20386870205402374\n",
      "Episode 3. Training Carried out at step = 10. Actorloss = -0.0010142545215785503, Critic_loss = 4.1873087882995605\n",
      "Episode 3. Training Carried out at step = 20. Actorloss = -0.0006924434565007687, Critic_loss = 17.133798599243164\n",
      "Episode 3. Training Carried out at step = 30. Actorloss = -0.00030780036468058825, Critic_loss = 21.205991744995117\n",
      "Episode 3. Training Carried out at step = 40. Actorloss = 8.888739830581471e-05, Critic_loss = 2.69667387008667\n",
      "Episode 3. Training Carried out at step = 50. Actorloss = 0.000514100247528404, Critic_loss = 0.34407490491867065\n",
      "Episode 3. Training Carried out at step = 60. Actorloss = 0.0009668027050793171, Critic_loss = 0.42905929684638977\n",
      "Episode 3. Training Carried out at step = 70. Actorloss = 0.001418133033439517, Critic_loss = 0.4082326591014862\n",
      "Episode 3. Training Carried out at step = 80. Actorloss = 0.0018862002762034535, Critic_loss = 0.34793055057525635\n",
      "Episode 3. Training Carried out at step = 90. Actorloss = 0.0023611432407051325, Critic_loss = 0.2905822694301605\n",
      "Episode 3. Training Carried out at step = 100. Actorloss = 0.0028227088041603565, Critic_loss = 0.2828774154186249\n",
      "Episode 3. Training Carried out at step = 110. Actorloss = 0.003278633812442422, Critic_loss = 0.2682933211326599\n",
      "Episode 3. Training Carried out at step = 120. Actorloss = 0.0037254421040415764, Critic_loss = 0.24830852448940277\n",
      "Episode 3. Training Carried out at step = 130. Actorloss = 0.004160079173743725, Critic_loss = 0.22470249235630035\n",
      "Episode 3. Training Carried out at step = 140. Actorloss = 0.0046021356247365475, Critic_loss = 0.19869454205036163\n",
      "Episode 3. Training Carried out at step = 150. Actorloss = 0.005024394486099482, Critic_loss = 0.17158901691436768\n",
      "Episode 3. Training Carried out at step = 160. Actorloss = 0.005392295774072409, Critic_loss = 0.14453072845935822\n",
      "Episode 3. Training Carried out at step = 170. Actorloss = 0.0057194968685507774, Critic_loss = 0.11601749807596207\n",
      "Episode 3. Training Carried out at step = 180. Actorloss = 0.005967098753899336, Critic_loss = 0.10896019637584686\n",
      "Episode 3. Training Carried out at step = 190. Actorloss = 0.006267447490245104, Critic_loss = 0.0749019905924797\n",
      "Episode 3. Training Carried out at step = 200. Actorloss = 0.006585980299860239, Critic_loss = 0.021354615688323975\n",
      "Episode 3. Training Carried out at step = 210. Actorloss = 0.006836164277046919, Critic_loss = 0.06447823345661163\n",
      "Episode 3. Training Carried out at step = 220. Actorloss = 0.007053741253912449, Critic_loss = 0.1419515758752823\n",
      "Episode 3. Training Carried out at step = 230. Actorloss = 0.007275135722011328, Critic_loss = 0.19774870574474335\n",
      "Episode 3. Training Carried out at step = 240. Actorloss = 0.00749730970710516, Critic_loss = 0.21982675790786743\n",
      "Episode 3. Training Carried out at step = 250. Actorloss = 0.007783778943121433, Critic_loss = 0.08118239790201187\n",
      "Episode 3. Training Carried out at step = 260. Actorloss = 0.008008812554180622, Critic_loss = 0.056658387184143066\n",
      "Episode 3. Training Carried out at step = 270. Actorloss = 0.00819430686533451, Critic_loss = 0.03238248452544212\n",
      "Episode 3. Training Carried out at step = 280. Actorloss = 0.008437528274953365, Critic_loss = 0.5329422950744629\n",
      "Episode 3. Training Carried out at step = 290. Actorloss = 0.008663034997880459, Critic_loss = 0.8951290845870972\n",
      "Episode 3. Training Carried out at step = 300. Actorloss = 0.008901388384401798, Critic_loss = 1.2708513736724854\n",
      "Episode 3. Training Carried out at step = 310. Actorloss = 0.009151408448815346, Critic_loss = 1.4763294458389282\n",
      "Episode 3. Training Carried out at step = 320. Actorloss = 0.009403758682310581, Critic_loss = 1.4678643941879272\n",
      "Episode 3. Training Carried out at step = 330. Actorloss = 0.009702703915536404, Critic_loss = 1.3717182874679565\n",
      "Episode 3. Training Carried out at step = 340. Actorloss = 0.010001830756664276, Critic_loss = 0.9069623351097107\n",
      "Episode 3. Training Carried out at step = 350. Actorloss = 0.010319272056221962, Critic_loss = 0.38508284091949463\n",
      "Episode 3. Training Carried out at step = 360. Actorloss = 0.01065275352448225, Critic_loss = 0.0739348828792572\n",
      "Episode 3. Training Carried out at step = 370. Actorloss = 0.010998949408531189, Critic_loss = 0.27559739351272583\n",
      "Episode 3. Training Carried out at step = 380. Actorloss = 0.011379024013876915, Critic_loss = 0.30098170042037964\n",
      "Episode 3. Training Carried out at step = 390. Actorloss = 0.01177034992724657, Critic_loss = 0.2952798008918762\n",
      "Episode 3. Training Carried out at step = 400. Actorloss = 0.01219155266880989, Critic_loss = 0.22087928652763367\n",
      "Episode 3. Training Carried out at step = 410. Actorloss = 0.012589126825332642, Critic_loss = 0.206231027841568\n",
      "Episode 3. Training Carried out at step = 420. Actorloss = 0.012994308024644852, Critic_loss = 0.2026395946741104\n",
      "Episode 3. Training Carried out at step = 430. Actorloss = 0.013382728211581707, Critic_loss = 0.19379495084285736\n",
      "Episode 3. Training Carried out at step = 440. Actorloss = 0.013771632686257362, Critic_loss = 0.18103837966918945\n",
      "Episode 3. Training Carried out at step = 450. Actorloss = 0.014143110252916813, Critic_loss = 0.16474665701389313\n",
      "Episode 3. Training Carried out at step = 460. Actorloss = 0.014498237520456314, Critic_loss = 0.14655885100364685\n",
      "Episode 3. Training Carried out at step = 470. Actorloss = 0.014842081815004349, Critic_loss = 0.12735475599765778\n",
      "Episode 3. Training Carried out at step = 480. Actorloss = 0.015168978832662106, Critic_loss = 0.10808809101581573\n",
      "Episode 3. Training Carried out at step = 490. Actorloss = 0.015354074537754059, Critic_loss = 0.0905732586979866\n",
      "Episode 3. Training Carried out at step = 500. Actorloss = 0.015641584992408752, Critic_loss = 0.07875627279281616\n",
      "Episode 3. Training Carried out at step = 510. Actorloss = 0.015906667336821556, Critic_loss = 0.02688578888773918\n",
      "Episode 3. Training Carried out at step = 520. Actorloss = 0.016205262392759323, Critic_loss = 0.033925313502550125\n",
      "Episode 3. Training Carried out at step = 530. Actorloss = 0.016423970460891724, Critic_loss = 0.10347843170166016\n",
      "Episode 3. Training Carried out at step = 540. Actorloss = 0.016621502116322517, Critic_loss = 0.18294280767440796\n",
      "Episode 3. Training Carried out at step = 550. Actorloss = 0.016832344233989716, Critic_loss = 0.22925767302513123\n",
      "Episode 3. Training Carried out at step = 560. Actorloss = 0.0170968696475029, Critic_loss = 0.14102298021316528\n",
      "Episode 3. Training Carried out at step = 570. Actorloss = 0.01734812743961811, Critic_loss = 0.07122422754764557\n",
      "Episode 3. Training Carried out at step = 580. Actorloss = 0.017552809789776802, Critic_loss = 0.06066003814339638\n",
      "Episode 3. Training Carried out at step = 590. Actorloss = 0.01772075518965721, Critic_loss = 0.12368745356798172\n",
      "Episode 3. Training Carried out at step = 600. Actorloss = 0.017950423061847687, Critic_loss = 0.7698186635971069\n",
      "Episode 3. Training Carried out at step = 610. Actorloss = 0.018182069063186646, Critic_loss = 1.1666302680969238\n",
      "Episode 3. Training Carried out at step = 620. Actorloss = 0.01840951479971409, Critic_loss = 1.6041654348373413\n",
      "Episode 3. Training Carried out at step = 630. Actorloss = 0.018653245642781258, Critic_loss = 1.6076953411102295\n",
      "Episode 3. Training Carried out at step = 640. Actorloss = 0.018915144726634026, Critic_loss = 1.5912491083145142\n",
      "Episode 3. Training Carried out at step = 650. Actorloss = 0.019196921959519386, Critic_loss = 1.2722628116607666\n",
      "Episode 3. Training Carried out at step = 660. Actorloss = 0.019487179815769196, Critic_loss = 0.6440674066543579\n",
      "Episode 3. Training Carried out at step = 670. Actorloss = 0.01979987882077694, Critic_loss = 0.15503151714801788\n",
      "Episode 3. Training Carried out at step = 680. Actorloss = 0.020117754116654396, Critic_loss = 0.17862588167190552\n",
      "Episode 3. Training Carried out at step = 690. Actorloss = 0.020471468567848206, Critic_loss = 0.2450888454914093\n",
      "Episode 3. Training Carried out at step = 700. Actorloss = 0.020838089287281036, Critic_loss = 0.2443361133337021\n",
      "Episode 3. Training Carried out at step = 710. Actorloss = 0.02122662402689457, Critic_loss = 0.20882278680801392\n",
      "Episode 3. Training Carried out at step = 720. Actorloss = 0.02162642404437065, Critic_loss = 0.15488407015800476\n",
      "Episode 3. Training Carried out at step = 730. Actorloss = 0.021994493901729584, Critic_loss = 0.15339575707912445\n",
      "Episode 3. Training Carried out at step = 740. Actorloss = 0.02237527072429657, Critic_loss = 0.14715270698070526\n",
      "Episode 3. Training Carried out at step = 750. Actorloss = 0.022740496322512627, Critic_loss = 0.1378927379846573\n",
      "Episode 3. Training Carried out at step = 760. Actorloss = 0.02311065047979355, Critic_loss = 0.1257379800081253\n",
      "Episode 3. Training Carried out at step = 770. Actorloss = 0.023452460765838623, Critic_loss = 0.11171377450227737\n",
      "Episode 3. Training Carried out at step = 780. Actorloss = 0.023783981800079346, Critic_loss = 0.09684865176677704\n",
      "Episode 3. Training Carried out at step = 790. Actorloss = 0.024089142680168152, Critic_loss = 0.08158466219902039\n",
      "Episode 3. Training Carried out at step = 800. Actorloss = 0.02430572919547558, Critic_loss = 0.0667518749833107\n",
      "Episode 3. Training Carried out at step = 810. Actorloss = 0.02450384758412838, Critic_loss = 0.053978729993104935\n",
      "Episode 3. Training Carried out at step = 820. Actorloss = 0.024780642241239548, Critic_loss = 0.024013886228203773\n",
      "Episode 3. Training Carried out at step = 830. Actorloss = 0.025045787915587425, Critic_loss = 0.016698084771633148\n",
      "Episode 3. Training Carried out at step = 840. Actorloss = 0.02532694861292839, Critic_loss = 0.08689159154891968\n",
      "Episode 3. Training Carried out at step = 850. Actorloss = 0.025502298027276993, Critic_loss = 0.17917178571224213\n",
      "Episode 3. Training Carried out at step = 860. Actorloss = 0.02571045607328415, Critic_loss = 0.2515963017940521\n",
      "Episode 3. Training Carried out at step = 870. Actorloss = 0.025944892317056656, Critic_loss = 0.21952298283576965\n",
      "Episode 3. Training Carried out at step = 880. Actorloss = 0.026247071102261543, Critic_loss = 0.0965963751077652\n",
      "Episode 3. Training Carried out at step = 890. Actorloss = 0.026458119973540306, Critic_loss = 0.08427077531814575\n",
      "Episode 3. Training Carried out at step = 900. Actorloss = 0.026646655052900314, Critic_loss = 0.06643888354301453\n",
      "Episode 3. Training Carried out at step = 910. Actorloss = 0.02680850215256214, Critic_loss = 0.36639881134033203\n",
      "Episode 3. Training Carried out at step = 920. Actorloss = 0.027047086507081985, Critic_loss = 1.1915297508239746\n",
      "Episode 3. Training Carried out at step = 930. Actorloss = 0.02728012576699257, Critic_loss = 1.5380210876464844\n",
      "Episode 3. Training Carried out at step = 940. Actorloss = 0.02750755473971367, Critic_loss = 1.7639333009719849\n",
      "Episode 3. Training Carried out at step = 950. Actorloss = 0.02776438742876053, Critic_loss = 1.8125343322753906\n",
      "Episode 3. Training Carried out at step = 960. Actorloss = 0.028025364503264427, Critic_loss = 1.5811173915863037\n",
      "Episode 3. Training Carried out at step = 970. Actorloss = 0.028302032500505447, Critic_loss = 0.9776952266693115\n",
      "Episode 3. Training Carried out at step = 980. Actorloss = 0.028599971905350685, Critic_loss = 0.3249157667160034\n",
      "Episode 3. Training Carried out at step = 990. Actorloss = 0.028888799250125885, Critic_loss = 0.10019589215517044\n",
      "Episode 3. Training Carried out at step = 1000. Actorloss = 0.029216093942523003, Critic_loss = 0.20434841513633728\n",
      "Episode 4. Training Carried out at step = 10. Actorloss = 0.0272003673017025, Critic_loss = 4.107243537902832\n",
      "Episode 4. Training Carried out at step = 20. Actorloss = 0.02758929319679737, Critic_loss = 16.859697341918945\n",
      "Episode 4. Training Carried out at step = 30. Actorloss = 0.027860239148139954, Critic_loss = 21.934551239013672\n",
      "Episode 4. Training Carried out at step = 40. Actorloss = 0.028325892984867096, Critic_loss = 3.717557430267334\n",
      "Episode 4. Training Carried out at step = 50. Actorloss = 0.02875479683279991, Critic_loss = 0.3745053708553314\n",
      "Episode 4. Training Carried out at step = 60. Actorloss = 0.029188139364123344, Critic_loss = 0.42454251646995544\n",
      "Episode 4. Training Carried out at step = 70. Actorloss = 0.02963494509458542, Critic_loss = 0.4176729619503021\n",
      "Episode 4. Training Carried out at step = 80. Actorloss = 0.030093004927039146, Critic_loss = 0.35469016432762146\n",
      "Episode 4. Training Carried out at step = 90. Actorloss = 0.03058708645403385, Critic_loss = 0.2922283411026001\n",
      "Episode 4. Training Carried out at step = 100. Actorloss = 0.031044401228427887, Critic_loss = 0.2860412299633026\n",
      "Episode 4. Training Carried out at step = 110. Actorloss = 0.031497400254011154, Critic_loss = 0.2743682563304901\n",
      "Episode 4. Training Carried out at step = 120. Actorloss = 0.03194313868880272, Critic_loss = 0.2576066553592682\n",
      "Episode 4. Training Carried out at step = 130. Actorloss = 0.03237910568714142, Critic_loss = 0.23739798367023468\n",
      "Episode 4. Training Carried out at step = 140. Actorloss = 0.0328306183218956, Critic_loss = 0.21462956070899963\n",
      "Episode 4. Training Carried out at step = 150. Actorloss = 0.03329101577401161, Critic_loss = 0.19067023694515228\n",
      "Episode 4. Training Carried out at step = 160. Actorloss = 0.03364419937133789, Critic_loss = 0.16644400358200073\n",
      "Episode 4. Training Carried out at step = 170. Actorloss = 0.03388907387852669, Critic_loss = 0.14473743736743927\n",
      "Episode 4. Training Carried out at step = 180. Actorloss = 0.034091416746377945, Critic_loss = 0.12315968424081802\n",
      "Episode 4. Training Carried out at step = 190. Actorloss = 0.03444022685289383, Critic_loss = 0.08622869849205017\n",
      "Episode 4. Training Carried out at step = 200. Actorloss = 0.0347713902592659, Critic_loss = 0.01606753282248974\n",
      "Episode 4. Training Carried out at step = 210. Actorloss = 0.03506097570061684, Critic_loss = 0.047572147101163864\n",
      "Episode 4. Training Carried out at step = 220. Actorloss = 0.035307664424180984, Critic_loss = 0.11618306487798691\n",
      "Episode 4. Training Carried out at step = 230. Actorloss = 0.03548552840948105, Critic_loss = 0.17104481160640717\n",
      "Episode 4. Training Carried out at step = 240. Actorloss = 0.03576934337615967, Critic_loss = 0.16225828230381012\n",
      "Episode 4. Training Carried out at step = 250. Actorloss = 0.03612852096557617, Critic_loss = 0.04701164737343788\n",
      "Episode 4. Training Carried out at step = 260. Actorloss = 0.03635476157069206, Critic_loss = 0.03831108286976814\n",
      "Episode 4. Training Carried out at step = 270. Actorloss = 0.03653620555996895, Critic_loss = 0.0199963990598917\n",
      "Episode 4. Training Carried out at step = 280. Actorloss = 0.03674127906560898, Critic_loss = 0.32447049021720886\n",
      "Episode 4. Training Carried out at step = 290. Actorloss = 0.03702366724610329, Critic_loss = 1.0662895441055298\n",
      "Episode 4. Training Carried out at step = 300. Actorloss = 0.03725086525082588, Critic_loss = 1.42726469039917\n",
      "Episode 4. Training Carried out at step = 310. Actorloss = 0.03749014809727669, Critic_loss = 1.6222200393676758\n",
      "Episode 4. Training Carried out at step = 320. Actorloss = 0.03774050995707512, Critic_loss = 1.5396960973739624\n",
      "Episode 4. Training Carried out at step = 330. Actorloss = 0.03802211955189705, Critic_loss = 1.4080188274383545\n",
      "Episode 4. Training Carried out at step = 340. Actorloss = 0.03830887749791145, Critic_loss = 0.8326331377029419\n",
      "Episode 4. Training Carried out at step = 350. Actorloss = 0.038608700037002563, Critic_loss = 0.2736896574497223\n",
      "Episode 4. Training Carried out at step = 360. Actorloss = 0.03888911008834839, Critic_loss = 0.15669509768486023\n",
      "Episode 4. Training Carried out at step = 370. Actorloss = 0.03921106830239296, Critic_loss = 0.27859392762184143\n",
      "Episode 4. Training Carried out at step = 380. Actorloss = 0.0395609475672245, Critic_loss = 0.2733456492424011\n",
      "Episode 4. Training Carried out at step = 390. Actorloss = 0.039900120347738266, Critic_loss = 0.2552313208580017\n",
      "Episode 4. Training Carried out at step = 400. Actorloss = 0.04030117020010948, Critic_loss = 0.17072680592536926\n",
      "Episode 4. Training Carried out at step = 410. Actorloss = 0.04066072031855583, Critic_loss = 0.16289015114307404\n",
      "Episode 4. Training Carried out at step = 420. Actorloss = 0.041015733033418655, Critic_loss = 0.1565689891576767\n",
      "Episode 4. Training Carried out at step = 430. Actorloss = 0.0413617379963398, Critic_loss = 0.14714530110359192\n",
      "Episode 4. Training Carried out at step = 440. Actorloss = 0.041690289974212646, Critic_loss = 0.13527698814868927\n",
      "Episode 4. Training Carried out at step = 450. Actorloss = 0.042008839547634125, Critic_loss = 0.12178360670804977\n",
      "Episode 4. Training Carried out at step = 460. Actorloss = 0.04231272637844086, Critic_loss = 0.10717549175024033\n",
      "Episode 4. Training Carried out at step = 470. Actorloss = 0.042615510523319244, Critic_loss = 0.09271492809057236\n",
      "Episode 4. Training Carried out at step = 480. Actorloss = 0.042843177914619446, Critic_loss = 0.0755491629242897\n",
      "Episode 4. Training Carried out at step = 490. Actorloss = 0.042930975556373596, Critic_loss = 0.06630527973175049\n",
      "Episode 4. Training Carried out at step = 500. Actorloss = 0.043195806443691254, Critic_loss = 0.0473734587430954\n",
      "Episode 4. Training Carried out at step = 510. Actorloss = 0.04340561851859093, Critic_loss = 0.01216940674930811\n",
      "Episode 4. Training Carried out at step = 520. Actorloss = 0.04378821328282356, Critic_loss = 0.049833595752716064\n",
      "Episode 4. Training Carried out at step = 530. Actorloss = 0.043943602591753006, Critic_loss = 0.12954570353031158\n",
      "Episode 4. Training Carried out at step = 540. Actorloss = 0.04412426799535751, Critic_loss = 0.2104247808456421\n",
      "Episode 4. Training Carried out at step = 550. Actorloss = 0.04428983852267265, Critic_loss = 0.2563440799713135\n",
      "Episode 4. Training Carried out at step = 560. Actorloss = 0.04463197663426399, Critic_loss = 0.1111440435051918\n",
      "Episode 4. Training Carried out at step = 570. Actorloss = 0.04483392834663391, Critic_loss = 0.0778505876660347\n",
      "Episode 4. Training Carried out at step = 580. Actorloss = 0.04503024369478226, Critic_loss = 0.06760317087173462\n",
      "Episode 4. Training Carried out at step = 590. Actorloss = 0.04508482664823532, Critic_loss = 0.05888925865292549\n",
      "Episode 4. Training Carried out at step = 600. Actorloss = 0.04534541442990303, Critic_loss = 0.9347220659255981\n",
      "Episode 4. Training Carried out at step = 610. Actorloss = 0.045541275292634964, Critic_loss = 1.5017451047897339\n",
      "Episode 4. Training Carried out at step = 620. Actorloss = 0.04571899399161339, Critic_loss = 1.7239967584609985\n",
      "Episode 4. Training Carried out at step = 630. Actorloss = 0.04593552276492119, Critic_loss = 1.7608695030212402\n",
      "Episode 4. Training Carried out at step = 640. Actorloss = 0.04616442695260048, Critic_loss = 1.6638691425323486\n",
      "Episode 4. Training Carried out at step = 650. Actorloss = 0.046403076499700546, Critic_loss = 1.2346729040145874\n",
      "Episode 4. Training Carried out at step = 660. Actorloss = 0.04664776101708412, Critic_loss = 0.5443911552429199\n",
      "Episode 4. Training Carried out at step = 670. Actorloss = 0.04688569903373718, Critic_loss = 0.10147541761398315\n",
      "Episode 4. Training Carried out at step = 680. Actorloss = 0.04713808372616768, Critic_loss = 0.2031688243150711\n",
      "Episode 4. Training Carried out at step = 690. Actorloss = 0.047436147928237915, Critic_loss = 0.2109859138727188\n",
      "Episode 4. Training Carried out at step = 700. Actorloss = 0.047743793576955795, Critic_loss = 0.2040272355079651\n",
      "Episode 4. Training Carried out at step = 710. Actorloss = 0.04807628318667412, Critic_loss = 0.16844072937965393\n",
      "Episode 4. Training Carried out at step = 720. Actorloss = 0.048409122973680496, Critic_loss = 0.11835942417383194\n",
      "Episode 4. Training Carried out at step = 730. Actorloss = 0.04873405769467354, Critic_loss = 0.1162230372428894\n",
      "Episode 4. Training Carried out at step = 740. Actorloss = 0.04905970022082329, Critic_loss = 0.11155585199594498\n",
      "Episode 4. Training Carried out at step = 750. Actorloss = 0.049352873116731644, Critic_loss = 0.10438429564237595\n",
      "Episode 4. Training Carried out at step = 760. Actorloss = 0.049658313393592834, Critic_loss = 0.09545871615409851\n",
      "Episode 4. Training Carried out at step = 770. Actorloss = 0.04994066804647446, Critic_loss = 0.08538778126239777\n",
      "Episode 4. Training Carried out at step = 780. Actorloss = 0.05021127685904503, Critic_loss = 0.07470240443944931\n",
      "Episode 4. Training Carried out at step = 790. Actorloss = 0.050474852323532104, Critic_loss = 0.0638829842209816\n",
      "Episode 4. Training Carried out at step = 800. Actorloss = 0.05051228031516075, Critic_loss = 0.05632447823882103\n",
      "Episode 4. Training Carried out at step = 810. Actorloss = 0.05069804564118385, Critic_loss = 0.04231038689613342\n",
      "Episode 4. Training Carried out at step = 820. Actorloss = 0.050937436521053314, Critic_loss = 0.012181366793811321\n",
      "Episode 4. Training Carried out at step = 830. Actorloss = 0.051183782517910004, Critic_loss = 0.023335043340921402\n",
      "Episode 4. Training Carried out at step = 840. Actorloss = 0.05147620663046837, Critic_loss = 0.08661536127328873\n",
      "Episode 4. Training Carried out at step = 850. Actorloss = 0.051620788872241974, Critic_loss = 0.18411162495613098\n",
      "Episode 4. Training Carried out at step = 860. Actorloss = 0.05178635194897652, Critic_loss = 0.2542896270751953\n",
      "Episode 4. Training Carried out at step = 870. Actorloss = 0.052042484283447266, Critic_loss = 0.18181943893432617\n",
      "Episode 4. Training Carried out at step = 880. Actorloss = 0.05227163806557655, Critic_loss = 0.08907029032707214\n",
      "Episode 4. Training Carried out at step = 890. Actorloss = 0.05243021622300148, Critic_loss = 0.07815221697092056\n",
      "Episode 4. Training Carried out at step = 900. Actorloss = 0.05258195102214813, Critic_loss = 0.06606179475784302\n",
      "Episode 4. Training Carried out at step = 910. Actorloss = 0.05266474932432175, Critic_loss = 0.2779824137687683\n",
      "Episode 4. Training Carried out at step = 920. Actorloss = 0.05288813263177872, Critic_loss = 1.3535454273223877\n",
      "Episode 4. Training Carried out at step = 930. Actorloss = 0.05305571109056473, Critic_loss = 1.6035484075546265\n",
      "Episode 4. Training Carried out at step = 940. Actorloss = 0.053229354321956635, Critic_loss = 1.7840744256973267\n",
      "Episode 4. Training Carried out at step = 950. Actorloss = 0.053424179553985596, Critic_loss = 1.845823884010315\n",
      "Episode 4. Training Carried out at step = 960. Actorloss = 0.053631316870450974, Critic_loss = 1.5317037105560303\n",
      "Episode 4. Training Carried out at step = 970. Actorloss = 0.0538463369011879, Critic_loss = 0.8289317488670349\n",
      "Episode 4. Training Carried out at step = 980. Actorloss = 0.05407246947288513, Critic_loss = 0.19567754864692688\n",
      "Episode 4. Training Carried out at step = 990. Actorloss = 0.05430891737341881, Critic_loss = 0.14569558203220367\n",
      "Episode 4. Training Carried out at step = 1000. Actorloss = 0.05456792563199997, Critic_loss = 0.20806610584259033\n",
      "Episode 4. Model checkpoint is saved \n",
      "\n",
      "Episode 4. tf.summary -> Tensorboard is updated \n",
      "\n",
      "Episode 5. Training Carried out at step = 10. Actorloss = 0.05122625455260277, Critic_loss = 4.216875076293945\n",
      "Episode 5. Training Carried out at step = 20. Actorloss = 0.0516703762114048, Critic_loss = 16.977754592895508\n",
      "Episode 5. Training Carried out at step = 30. Actorloss = 0.051952067762613297, Critic_loss = 22.077342987060547\n",
      "Episode 5. Training Carried out at step = 40. Actorloss = 0.0523538663983345, Critic_loss = 3.9163365364074707\n",
      "Episode 5. Training Carried out at step = 50. Actorloss = 0.05271414667367935, Critic_loss = 0.328612744808197\n",
      "Episode 5. Training Carried out at step = 60. Actorloss = 0.0531037375330925, Critic_loss = 0.34020906686782837\n",
      "Episode 5. Training Carried out at step = 70. Actorloss = 0.053516190499067307, Critic_loss = 0.33152610063552856\n",
      "Episode 5. Training Carried out at step = 80. Actorloss = 0.05394219607114792, Critic_loss = 0.2692960500717163\n",
      "Episode 5. Training Carried out at step = 90. Actorloss = 0.05441514402627945, Critic_loss = 0.22421875596046448\n",
      "Episode 5. Training Carried out at step = 100. Actorloss = 0.054838284850120544, Critic_loss = 0.22006499767303467\n",
      "Episode 5. Training Carried out at step = 110. Actorloss = 0.05525811389088631, Critic_loss = 0.21173693239688873\n",
      "Episode 5. Training Carried out at step = 120. Actorloss = 0.05567222461104393, Critic_loss = 0.20000021159648895\n",
      "Episode 5. Training Carried out at step = 130. Actorloss = 0.05607854574918747, Critic_loss = 0.1856580376625061\n",
      "Episode 5. Training Carried out at step = 140. Actorloss = 0.056515056639909744, Critic_loss = 0.1696433573961258\n",
      "Episode 5. Training Carried out at step = 150. Actorloss = 0.056963540613651276, Critic_loss = 0.1521829068660736\n",
      "Episode 5. Training Carried out at step = 160. Actorloss = 0.057271383702754974, Critic_loss = 0.13451996445655823\n",
      "Episode 5. Training Carried out at step = 170. Actorloss = 0.05743342638015747, Critic_loss = 0.1201675534248352\n",
      "Episode 5. Training Carried out at step = 180. Actorloss = 0.05760054662823677, Critic_loss = 0.10363274067640305\n",
      "Episode 5. Training Carried out at step = 190. Actorloss = 0.057864315807819366, Critic_loss = 0.05986718088388443\n",
      "Episode 5. Training Carried out at step = 200. Actorloss = 0.05819772928953171, Critic_loss = 0.014422907494008541\n",
      "Episode 5. Training Carried out at step = 210. Actorloss = 0.0585346445441246, Critic_loss = 0.05620464310050011\n",
      "Episode 5. Training Carried out at step = 220. Actorloss = 0.05872325599193573, Critic_loss = 0.12464751303195953\n",
      "Episode 5. Training Carried out at step = 230. Actorloss = 0.05889648571610451, Critic_loss = 0.17795097827911377\n",
      "Episode 5. Training Carried out at step = 240. Actorloss = 0.05914418771862984, Critic_loss = 0.16893459856510162\n",
      "Episode 5. Training Carried out at step = 250. Actorloss = 0.059590261429548264, Critic_loss = 0.04848897457122803\n",
      "Episode 5. Training Carried out at step = 260. Actorloss = 0.05980619043111801, Critic_loss = 0.040217455476522446\n",
      "Episode 5. Training Carried out at step = 270. Actorloss = 0.05991803854703903, Critic_loss = 0.02122991904616356\n",
      "Episode 5. Training Carried out at step = 280. Actorloss = 0.060053955763578415, Critic_loss = 0.3289845287799835\n",
      "Episode 5. Training Carried out at step = 290. Actorloss = 0.06032533571124077, Critic_loss = 1.1073920726776123\n",
      "Episode 5. Training Carried out at step = 300. Actorloss = 0.06051550433039665, Critic_loss = 1.4740973711013794\n",
      "Episode 5. Training Carried out at step = 310. Actorloss = 0.06073479726910591, Critic_loss = 1.664360761642456\n",
      "Episode 5. Training Carried out at step = 320. Actorloss = 0.06097792088985443, Critic_loss = 1.5547336339950562\n",
      "Episode 5. Training Carried out at step = 330. Actorloss = 0.06123826652765274, Critic_loss = 1.4356658458709717\n",
      "Episode 5. Training Carried out at step = 340. Actorloss = 0.061496198177337646, Critic_loss = 0.8507668375968933\n",
      "Episode 5. Training Carried out at step = 350. Actorloss = 0.06174255907535553, Critic_loss = 0.28323546051979065\n",
      "Episode 5. Training Carried out at step = 360. Actorloss = 0.06199641153216362, Critic_loss = 0.14975209534168243\n",
      "Episode 5. Training Carried out at step = 370. Actorloss = 0.06227808818221092, Critic_loss = 0.2486324906349182\n",
      "Episode 5. Training Carried out at step = 380. Actorloss = 0.06257586181163788, Critic_loss = 0.24156787991523743\n",
      "Episode 5. Training Carried out at step = 390. Actorloss = 0.06291376799345016, Critic_loss = 0.22281856834888458\n",
      "Episode 5. Training Carried out at step = 400. Actorloss = 0.06327969580888748, Critic_loss = 0.1456533819437027\n",
      "Episode 5. Training Carried out at step = 410. Actorloss = 0.06361895054578781, Critic_loss = 0.13827216625213623\n",
      "Episode 5. Training Carried out at step = 420. Actorloss = 0.06391169130802155, Critic_loss = 0.1335218846797943\n",
      "Episode 5. Training Carried out at step = 430. Actorloss = 0.0642300471663475, Critic_loss = 0.12607388198375702\n",
      "Episode 5. Training Carried out at step = 440. Actorloss = 0.06452099233865738, Critic_loss = 0.11703149974346161\n",
      "Episode 5. Training Carried out at step = 450. Actorloss = 0.06481033563613892, Critic_loss = 0.10651659220457077\n",
      "Episode 5. Training Carried out at step = 460. Actorloss = 0.06510801613330841, Critic_loss = 0.09506015479564667\n",
      "Episode 5. Training Carried out at step = 470. Actorloss = 0.06537319719791412, Critic_loss = 0.08334152400493622\n",
      "Episode 5. Training Carried out at step = 480. Actorloss = 0.06555476039648056, Critic_loss = 0.06937777251005173\n",
      "Episode 5. Training Carried out at step = 490. Actorloss = 0.0656060203909874, Critic_loss = 0.062286995351314545\n",
      "Episode 5. Training Carried out at step = 500. Actorloss = 0.06581871956586838, Critic_loss = 0.04635292664170265\n",
      "Episode 5. Training Carried out at step = 510. Actorloss = 0.06603359431028366, Critic_loss = 0.014371243305504322\n",
      "Episode 5. Training Carried out at step = 520. Actorloss = 0.06647691130638123, Critic_loss = 0.04896124452352524\n",
      "Episode 5. Training Carried out at step = 530. Actorloss = 0.06661409139633179, Critic_loss = 0.1270637810230255\n",
      "Episode 5. Training Carried out at step = 540. Actorloss = 0.06678302586078644, Critic_loss = 0.21465516090393066\n",
      "Episode 5. Training Carried out at step = 550. Actorloss = 0.06692791730165482, Critic_loss = 0.2549161911010742\n",
      "Episode 5. Training Carried out at step = 560. Actorloss = 0.06733942031860352, Critic_loss = 0.10260286927223206\n",
      "Episode 5. Training Carried out at step = 570. Actorloss = 0.06748862564563751, Critic_loss = 0.0753028467297554\n",
      "Episode 5. Training Carried out at step = 580. Actorloss = 0.06770046800374985, Critic_loss = 0.0653000995516777\n",
      "Episode 5. Training Carried out at step = 590. Actorloss = 0.0676872655749321, Critic_loss = 0.058224424719810486\n",
      "Episode 5. Training Carried out at step = 600. Actorloss = 0.06796612590551376, Critic_loss = 0.9465644359588623\n",
      "Episode 5. Training Carried out at step = 610. Actorloss = 0.0681300163269043, Critic_loss = 1.5169649124145508\n",
      "Episode 5. Training Carried out at step = 620. Actorloss = 0.06831569224596024, Critic_loss = 1.7322571277618408\n",
      "Episode 5. Training Carried out at step = 630. Actorloss = 0.06851676106452942, Critic_loss = 1.777244210243225\n",
      "Episode 5. Training Carried out at step = 640. Actorloss = 0.06870882958173752, Critic_loss = 1.6789582967758179\n",
      "Episode 5. Training Carried out at step = 650. Actorloss = 0.06892470270395279, Critic_loss = 1.2365729808807373\n",
      "Episode 5. Training Carried out at step = 660. Actorloss = 0.0691225454211235, Critic_loss = 0.5429121255874634\n",
      "Episode 5. Training Carried out at step = 670. Actorloss = 0.06934673339128494, Critic_loss = 0.1031968742609024\n",
      "Episode 5. Training Carried out at step = 680. Actorloss = 0.0695619210600853, Critic_loss = 0.19097693264484406\n",
      "Episode 5. Training Carried out at step = 690. Actorloss = 0.0698309987783432, Critic_loss = 0.19463752210140228\n",
      "Episode 5. Training Carried out at step = 700. Actorloss = 0.07009643316268921, Critic_loss = 0.18619132041931152\n",
      "Episode 5. Training Carried out at step = 710. Actorloss = 0.07036223262548447, Critic_loss = 0.15156066417694092\n",
      "Episode 5. Training Carried out at step = 720. Actorloss = 0.07066091150045395, Critic_loss = 0.10424061864614487\n",
      "Episode 5. Training Carried out at step = 730. Actorloss = 0.07091821730136871, Critic_loss = 0.10288228839635849\n",
      "Episode 5. Training Carried out at step = 740. Actorloss = 0.07116873562335968, Critic_loss = 0.09898553788661957\n",
      "Episode 5. Training Carried out at step = 750. Actorloss = 0.07140494883060455, Critic_loss = 0.09318594634532928\n",
      "Episode 5. Training Carried out at step = 760. Actorloss = 0.07164362818002701, Critic_loss = 0.08591014891862869\n",
      "Episode 5. Training Carried out at step = 770. Actorloss = 0.0718691349029541, Critic_loss = 0.07757274061441422\n",
      "Episode 5. Training Carried out at step = 780. Actorloss = 0.07208086550235748, Critic_loss = 0.06873510032892227\n",
      "Episode 5. Training Carried out at step = 790. Actorloss = 0.07228071987628937, Critic_loss = 0.05954094976186752\n",
      "Episode 5. Training Carried out at step = 800. Actorloss = 0.07243680208921432, Critic_loss = 0.05061214044690132\n",
      "Episode 5. Training Carried out at step = 810. Actorloss = 0.0726071447134018, Critic_loss = 0.04053662344813347\n",
      "Episode 5. Training Carried out at step = 820. Actorloss = 0.0727708712220192, Critic_loss = 0.011345881968736649\n",
      "Episode 5. Training Carried out at step = 830. Actorloss = 0.07292602211236954, Critic_loss = 0.022301189601421356\n",
      "Episode 5. Training Carried out at step = 840. Actorloss = 0.07306808978319168, Critic_loss = 0.08542425185441971\n",
      "Episode 5. Training Carried out at step = 850. Actorloss = 0.07319104671478271, Critic_loss = 0.17930224537849426\n",
      "Episode 5. Training Carried out at step = 860. Actorloss = 0.07330970466136932, Critic_loss = 0.24878956377506256\n",
      "Episode 5. Training Carried out at step = 870. Actorloss = 0.07342562824487686, Critic_loss = 0.17725232243537903\n",
      "Episode 5. Training Carried out at step = 880. Actorloss = 0.07354114204645157, Critic_loss = 0.08570783585309982\n",
      "Episode 5. Training Carried out at step = 890. Actorloss = 0.07365857809782028, Critic_loss = 0.07507430762052536\n",
      "Episode 5. Training Carried out at step = 900. Actorloss = 0.07377985119819641, Critic_loss = 0.06250301748514175\n",
      "Episode 5. Training Carried out at step = 910. Actorloss = 0.07390667498111725, Critic_loss = 0.2807571589946747\n",
      "Episode 5. Training Carried out at step = 920. Actorloss = 0.07404042035341263, Critic_loss = 1.3760204315185547\n",
      "Episode 5. Training Carried out at step = 930. Actorloss = 0.07418256253004074, Critic_loss = 1.613943338394165\n",
      "Episode 5. Training Carried out at step = 940. Actorloss = 0.07433436065912247, Critic_loss = 1.7879420518875122\n",
      "Episode 5. Training Carried out at step = 950. Actorloss = 0.07449695467948914, Critic_loss = 1.8352631330490112\n",
      "Episode 5. Training Carried out at step = 960. Actorloss = 0.07467140257358551, Critic_loss = 1.5423442125320435\n",
      "Episode 5. Training Carried out at step = 970. Actorloss = 0.07485870271921158, Critic_loss = 0.8244490027427673\n",
      "Episode 5. Training Carried out at step = 980. Actorloss = 0.07505973428487778, Critic_loss = 0.18285468220710754\n",
      "Episode 5. Training Carried out at step = 990. Actorloss = 0.07527533918619156, Critic_loss = 0.1410941779613495\n",
      "Episode 5. Training Carried out at step = 1000. Actorloss = 0.07550179213285446, Critic_loss = 0.19573166966438293\n",
      "Episode 6. Training Carried out at step = 10. Actorloss = 0.07215549796819687, Critic_loss = 4.385238170623779\n",
      "Episode 6. Training Carried out at step = 20. Actorloss = 0.07261643558740616, Critic_loss = 17.900705337524414\n",
      "Episode 6. Training Carried out at step = 30. Actorloss = 0.07294676452875137, Critic_loss = 22.45833969116211\n",
      "Episode 6. Training Carried out at step = 40. Actorloss = 0.07330604642629623, Critic_loss = 3.455360174179077\n",
      "Episode 6. Training Carried out at step = 50. Actorloss = 0.07364827394485474, Critic_loss = 0.24153922498226166\n",
      "Episode 6. Training Carried out at step = 60. Actorloss = 0.07406066358089447, Critic_loss = 0.30350229144096375\n",
      "Episode 6. Training Carried out at step = 70. Actorloss = 0.07446935027837753, Critic_loss = 0.28888192772865295\n",
      "Episode 6. Training Carried out at step = 80. Actorloss = 0.07488684356212616, Critic_loss = 0.2370702177286148\n",
      "Episode 6. Training Carried out at step = 90. Actorloss = 0.07536657899618149, Critic_loss = 0.18449586629867554\n",
      "Episode 6. Training Carried out at step = 100. Actorloss = 0.07578514516353607, Critic_loss = 0.18131224811077118\n",
      "Episode 6. Training Carried out at step = 110. Actorloss = 0.07620074599981308, Critic_loss = 0.17507177591323853\n",
      "Episode 6. Training Carried out at step = 120. Actorloss = 0.0766112208366394, Critic_loss = 0.16586834192276\n",
      "Episode 6. Training Carried out at step = 130. Actorloss = 0.0770147293806076, Critic_loss = 0.1545606106519699\n",
      "Episode 6. Training Carried out at step = 140. Actorloss = 0.07745575159788132, Critic_loss = 0.14158228039741516\n",
      "Episode 6. Training Carried out at step = 150. Actorloss = 0.07791046053171158, Critic_loss = 0.1273607611656189\n",
      "Episode 6. Training Carried out at step = 160. Actorloss = 0.07820209860801697, Critic_loss = 0.11340544372797012\n",
      "Episode 6. Training Carried out at step = 170. Actorloss = 0.07838466018438339, Critic_loss = 0.09945046156644821\n",
      "Episode 6. Training Carried out at step = 180. Actorloss = 0.07845156639814377, Critic_loss = 0.0892426148056984\n",
      "Episode 6. Training Carried out at step = 190. Actorloss = 0.07875041663646698, Critic_loss = 0.0552566796541214\n",
      "Episode 6. Training Carried out at step = 200. Actorloss = 0.07911364734172821, Critic_loss = 0.017176838591694832\n",
      "Episode 6. Training Carried out at step = 210. Actorloss = 0.07938328385353088, Critic_loss = 0.06059626117348671\n",
      "Episode 6. Training Carried out at step = 220. Actorloss = 0.07956309616565704, Critic_loss = 0.13149787485599518\n",
      "Episode 6. Training Carried out at step = 230. Actorloss = 0.07979555428028107, Critic_loss = 0.18828797340393066\n",
      "Episode 6. Training Carried out at step = 240. Actorloss = 0.07997328042984009, Critic_loss = 0.1958245187997818\n",
      "Episode 6. Training Carried out at step = 250. Actorloss = 0.0804557204246521, Critic_loss = 0.05579899251461029\n",
      "Episode 6. Training Carried out at step = 260. Actorloss = 0.08066309243440628, Critic_loss = 0.04698557034134865\n",
      "Episode 6. Training Carried out at step = 270. Actorloss = 0.08076668530702591, Critic_loss = 0.025047117844223976\n",
      "Episode 6. Training Carried out at step = 280. Actorloss = 0.08092150092124939, Critic_loss = 0.3977460265159607\n",
      "Episode 6. Training Carried out at step = 290. Actorloss = 0.08116203546524048, Critic_loss = 1.0386345386505127\n",
      "Episode 6. Training Carried out at step = 300. Actorloss = 0.08139627426862717, Critic_loss = 1.4469338655471802\n",
      "Episode 6. Training Carried out at step = 310. Actorloss = 0.08161559700965881, Critic_loss = 1.6748573780059814\n",
      "Episode 6. Training Carried out at step = 320. Actorloss = 0.08181876689195633, Critic_loss = 1.5826969146728516\n",
      "Episode 6. Training Carried out at step = 330. Actorloss = 0.08207250386476517, Critic_loss = 1.5162115097045898\n",
      "Episode 6. Training Carried out at step = 340. Actorloss = 0.08232671022415161, Critic_loss = 0.9658557772636414\n",
      "Episode 6. Training Carried out at step = 350. Actorloss = 0.08256377279758453, Critic_loss = 0.3868340253829956\n",
      "Episode 6. Training Carried out at step = 360. Actorloss = 0.08282863348722458, Critic_loss = 0.11021354049444199\n",
      "Episode 6. Training Carried out at step = 370. Actorloss = 0.08307800441980362, Critic_loss = 0.2077515721321106\n",
      "Episode 6. Training Carried out at step = 380. Actorloss = 0.08337944746017456, Critic_loss = 0.20303557813167572\n",
      "Episode 6. Training Carried out at step = 390. Actorloss = 0.08367510885000229, Critic_loss = 0.18801769614219666\n",
      "Episode 6. Training Carried out at step = 400. Actorloss = 0.08405476063489914, Critic_loss = 0.11867494881153107\n",
      "Episode 6. Training Carried out at step = 410. Actorloss = 0.08437580615282059, Critic_loss = 0.11340959370136261\n",
      "Episode 6. Training Carried out at step = 420. Actorloss = 0.08470173925161362, Critic_loss = 0.11050406843423843\n",
      "Episode 6. Training Carried out at step = 430. Actorloss = 0.08499905467033386, Critic_loss = 0.10567407310009003\n",
      "Episode 6. Training Carried out at step = 440. Actorloss = 0.08531662076711655, Critic_loss = 0.09870035946369171\n",
      "Episode 6. Training Carried out at step = 450. Actorloss = 0.08558263629674911, Critic_loss = 0.09052242338657379\n",
      "Episode 6. Training Carried out at step = 460. Actorloss = 0.08586063981056213, Critic_loss = 0.08158016949892044\n",
      "Episode 6. Training Carried out at step = 470. Actorloss = 0.0861305519938469, Critic_loss = 0.07198729366064072\n",
      "Episode 6. Training Carried out at step = 480. Actorloss = 0.08639451116323471, Critic_loss = 0.062164921313524246\n",
      "Episode 6. Training Carried out at step = 490. Actorloss = 0.08639877289533615, Critic_loss = 0.0569487027823925\n",
      "Episode 6. Training Carried out at step = 500. Actorloss = 0.08660054951906204, Critic_loss = 0.04482603073120117\n",
      "Episode 6. Training Carried out at step = 510. Actorloss = 0.08686356246471405, Critic_loss = 0.009932894259691238\n",
      "Episode 6. Training Carried out at step = 520. Actorloss = 0.0872117280960083, Critic_loss = 0.05376715958118439\n",
      "Episode 6. Training Carried out at step = 530. Actorloss = 0.08735966682434082, Critic_loss = 0.1320430338382721\n",
      "Episode 6. Training Carried out at step = 540. Actorloss = 0.08750993013381958, Critic_loss = 0.22014081478118896\n",
      "Episode 6. Training Carried out at step = 550. Actorloss = 0.08765821903944016, Critic_loss = 0.26023995876312256\n",
      "Episode 6. Training Carried out at step = 560. Actorloss = 0.08786290138959885, Critic_loss = 0.1324462890625\n",
      "Episode 6. Training Carried out at step = 570. Actorloss = 0.08801200985908508, Critic_loss = 0.08099167793989182\n",
      "Episode 6. Training Carried out at step = 580. Actorloss = 0.08815229684114456, Critic_loss = 0.07085787504911423\n",
      "Episode 6. Training Carried out at step = 590. Actorloss = 0.08828441798686981, Critic_loss = 0.06886736303567886\n",
      "Episode 6. Training Carried out at step = 600. Actorloss = 0.08844611793756485, Critic_loss = 1.0594784021377563\n",
      "Episode 6. Training Carried out at step = 610. Actorloss = 0.08860372751951218, Critic_loss = 1.3314955234527588\n",
      "Episode 6. Training Carried out at step = 620. Actorloss = 0.08876897394657135, Critic_loss = 1.731803297996521\n",
      "Episode 6. Training Carried out at step = 630. Actorloss = 0.08894439041614532, Critic_loss = 1.748232126235962\n",
      "Episode 6. Training Carried out at step = 640. Actorloss = 0.08913012593984604, Critic_loss = 1.723120093345642\n",
      "Episode 6. Training Carried out at step = 650. Actorloss = 0.08932743221521378, Critic_loss = 1.3299612998962402\n",
      "Episode 6. Training Carried out at step = 660. Actorloss = 0.08953718096017838, Critic_loss = 0.6462904214859009\n",
      "Episode 6. Training Carried out at step = 670. Actorloss = 0.0897606760263443, Critic_loss = 0.1432904303073883\n",
      "Episode 6. Training Carried out at step = 680. Actorloss = 0.0899982750415802, Critic_loss = 0.15520326793193817\n",
      "Episode 6. Training Carried out at step = 690. Actorloss = 0.0902465432882309, Critic_loss = 0.17489352822303772\n",
      "Episode 6. Training Carried out at step = 700. Actorloss = 0.09050226956605911, Critic_loss = 0.16875828802585602\n",
      "Episode 6. Training Carried out at step = 710. Actorloss = 0.09076260775327682, Critic_loss = 0.13802117109298706\n",
      "Episode 6. Training Carried out at step = 720. Actorloss = 0.09102485328912735, Critic_loss = 0.09390491992235184\n",
      "Episode 6. Training Carried out at step = 730. Actorloss = 0.09128673374652863, Critic_loss = 0.09330937266349792\n",
      "Episode 6. Training Carried out at step = 740. Actorloss = 0.09154607355594635, Critic_loss = 0.09068162739276886\n",
      "Episode 6. Training Carried out at step = 750. Actorloss = 0.0918009951710701, Critic_loss = 0.08607114851474762\n",
      "Episode 6. Training Carried out at step = 760. Actorloss = 0.09204975515604019, Critic_loss = 0.07974851131439209\n",
      "Episode 6. Training Carried out at step = 770. Actorloss = 0.09229081869125366, Critic_loss = 0.07246316224336624\n",
      "Episode 6. Training Carried out at step = 780. Actorloss = 0.09252280741930008, Critic_loss = 0.06458111107349396\n",
      "Episode 6. Training Carried out at step = 790. Actorloss = 0.09274455159902573, Critic_loss = 0.0561995692551136\n",
      "Episode 6. Training Carried out at step = 800. Actorloss = 0.0929548591375351, Critic_loss = 0.04724903404712677\n",
      "Episode 6. Training Carried out at step = 810. Actorloss = 0.09315283596515656, Critic_loss = 0.041993845254182816\n",
      "Episode 6. Training Carried out at step = 820. Actorloss = 0.0933377742767334, Critic_loss = 0.01735023222863674\n",
      "Episode 6. Training Carried out at step = 830. Actorloss = 0.09350978583097458, Critic_loss = 0.020943859592080116\n",
      "Episode 6. Training Carried out at step = 840. Actorloss = 0.09367111325263977, Critic_loss = 0.09014849364757538\n",
      "Episode 6. Training Carried out at step = 850. Actorloss = 0.09382493048906326, Critic_loss = 0.18660059571266174\n",
      "Episode 6. Training Carried out at step = 860. Actorloss = 0.09397395700216293, Critic_loss = 0.2587795555591583\n",
      "Episode 6. Training Carried out at step = 870. Actorloss = 0.09412068128585815, Critic_loss = 0.20491814613342285\n",
      "Episode 6. Training Carried out at step = 880. Actorloss = 0.09426730126142502, Critic_loss = 0.09530674666166306\n",
      "Episode 6. Training Carried out at step = 890. Actorloss = 0.09441577643156052, Critic_loss = 0.08344848453998566\n",
      "Episode 6. Training Carried out at step = 900. Actorloss = 0.09456793963909149, Critic_loss = 0.0662275105714798\n",
      "Episode 6. Training Carried out at step = 910. Actorloss = 0.09472557157278061, Critic_loss = 0.295915424823761\n",
      "Episode 6. Training Carried out at step = 920. Actorloss = 0.09488982707262039, Critic_loss = 1.3483167886734009\n",
      "Episode 6. Training Carried out at step = 930. Actorloss = 0.09506216645240784, Critic_loss = 1.5947034358978271\n",
      "Episode 6. Training Carried out at step = 940. Actorloss = 0.09524388611316681, Critic_loss = 1.8222403526306152\n",
      "Episode 6. Training Carried out at step = 950. Actorloss = 0.09543599933385849, Critic_loss = 1.8833657503128052\n",
      "Episode 6. Training Carried out at step = 960. Actorloss = 0.09563956409692764, Critic_loss = 1.5995131731033325\n",
      "Episode 6. Training Carried out at step = 970. Actorloss = 0.09585545212030411, Critic_loss = 0.93006432056427\n",
      "Episode 6. Training Carried out at step = 980. Actorloss = 0.09608456492424011, Critic_loss = 0.27860215306282043\n",
      "Episode 6. Training Carried out at step = 990. Actorloss = 0.09632766246795654, Critic_loss = 0.11298531293869019\n",
      "Episode 6. Training Carried out at step = 1000. Actorloss = 0.09658122807741165, Critic_loss = 0.16848796606063843\n",
      "Episode 7. Training Carried out at step = 10. Actorloss = 0.0935920923948288, Critic_loss = 3.89804744720459\n",
      "Episode 7. Training Carried out at step = 20. Actorloss = 0.0940033346414566, Critic_loss = 16.02974510192871\n",
      "Episode 7. Training Carried out at step = 30. Actorloss = 0.09478244930505753, Critic_loss = 19.483478546142578\n",
      "Episode 7. Training Carried out at step = 40. Actorloss = 0.09493864327669144, Critic_loss = 2.8300721645355225\n",
      "Episode 7. Training Carried out at step = 50. Actorloss = 0.09530866891145706, Critic_loss = 0.3877536356449127\n",
      "Episode 7. Training Carried out at step = 60. Actorloss = 0.09579106420278549, Critic_loss = 0.37170594930648804\n",
      "Episode 7. Training Carried out at step = 70. Actorloss = 0.09623956680297852, Critic_loss = 0.3636792302131653\n",
      "Episode 7. Training Carried out at step = 80. Actorloss = 0.09672266989946365, Critic_loss = 0.30480340123176575\n",
      "Episode 7. Training Carried out at step = 90. Actorloss = 0.09722260385751724, Critic_loss = 0.24682557582855225\n",
      "Episode 7. Training Carried out at step = 100. Actorloss = 0.09767910093069077, Critic_loss = 0.24306640028953552\n",
      "Episode 7. Training Carried out at step = 110. Actorloss = 0.0981324166059494, Critic_loss = 0.23611092567443848\n",
      "Episode 7. Training Carried out at step = 120. Actorloss = 0.09858058393001556, Critic_loss = 0.22545404732227325\n",
      "Episode 7. Training Carried out at step = 130. Actorloss = 0.0990217924118042, Critic_loss = 0.21258530020713806\n",
      "Episode 7. Training Carried out at step = 140. Actorloss = 0.09951063245534897, Critic_loss = 0.1975943148136139\n",
      "Episode 7. Training Carried out at step = 150. Actorloss = 0.09999561309814453, Critic_loss = 0.18137212097644806\n",
      "Episode 7. Training Carried out at step = 160. Actorloss = 0.10032414644956589, Critic_loss = 0.16465605795383453\n",
      "Episode 7. Training Carried out at step = 170. Actorloss = 0.10057448595762253, Critic_loss = 0.14337299764156342\n",
      "Episode 7. Training Carried out at step = 180. Actorloss = 0.10064201056957245, Critic_loss = 0.13958485424518585\n",
      "Episode 7. Training Carried out at step = 190. Actorloss = 0.10092563927173615, Critic_loss = 0.11370222270488739\n",
      "Episode 7. Training Carried out at step = 200. Actorloss = 0.10138726979494095, Critic_loss = 0.024624017998576164\n",
      "Episode 7. Training Carried out at step = 210. Actorloss = 0.10169817507266998, Critic_loss = 0.02602245658636093\n",
      "Episode 7. Training Carried out at step = 220. Actorloss = 0.10187461227178574, Critic_loss = 0.08145321905612946\n",
      "Episode 7. Training Carried out at step = 230. Actorloss = 0.10213969647884369, Critic_loss = 0.12395887821912766\n",
      "Episode 7. Training Carried out at step = 240. Actorloss = 0.10238490998744965, Critic_loss = 0.13794001936912537\n",
      "Episode 7. Training Carried out at step = 250. Actorloss = 0.10282527655363083, Critic_loss = 0.03158972039818764\n",
      "Episode 7. Training Carried out at step = 260. Actorloss = 0.10309690237045288, Critic_loss = 0.021764444187283516\n",
      "Episode 7. Training Carried out at step = 270. Actorloss = 0.10321935266256332, Critic_loss = 0.01921107992529869\n",
      "Episode 7. Training Carried out at step = 280. Actorloss = 0.10342995822429657, Critic_loss = 0.37840235233306885\n",
      "Episode 7. Training Carried out at step = 290. Actorloss = 0.10361979156732559, Critic_loss = 0.7148299813270569\n",
      "Episode 7. Training Carried out at step = 300. Actorloss = 0.10388664156198502, Critic_loss = 1.1475231647491455\n",
      "Episode 7. Training Carried out at step = 310. Actorloss = 0.10412072390317917, Critic_loss = 1.3436310291290283\n",
      "Episode 7. Training Carried out at step = 320. Actorloss = 0.1043572798371315, Critic_loss = 1.3582556247711182\n",
      "Episode 7. Training Carried out at step = 330. Actorloss = 0.10461506992578506, Critic_loss = 1.2590839862823486\n",
      "Episode 7. Training Carried out at step = 340. Actorloss = 0.10487974435091019, Critic_loss = 0.8493773341178894\n",
      "Episode 7. Training Carried out at step = 350. Actorloss = 0.10513825714588165, Critic_loss = 0.34075894951820374\n",
      "Episode 7. Training Carried out at step = 360. Actorloss = 0.10539042949676514, Critic_loss = 0.07550140470266342\n",
      "Episode 7. Training Carried out at step = 370. Actorloss = 0.10563958436250687, Critic_loss = 0.2775462865829468\n",
      "Episode 7. Training Carried out at step = 380. Actorloss = 0.1059504896402359, Critic_loss = 0.2830236554145813\n",
      "Episode 7. Training Carried out at step = 390. Actorloss = 0.10625138878822327, Critic_loss = 0.26792407035827637\n",
      "Episode 7. Training Carried out at step = 400. Actorloss = 0.10657376796007156, Critic_loss = 0.1920115202665329\n",
      "Episode 7. Training Carried out at step = 410. Actorloss = 0.10685840249061584, Critic_loss = 0.17671418190002441\n",
      "Episode 7. Training Carried out at step = 420. Actorloss = 0.10715684294700623, Critic_loss = 0.1741003543138504\n",
      "Episode 7. Training Carried out at step = 430. Actorloss = 0.10743308067321777, Critic_loss = 0.1684984713792801\n",
      "Episode 7. Training Carried out at step = 440. Actorloss = 0.1077067032456398, Critic_loss = 0.1605382263660431\n",
      "Episode 7. Training Carried out at step = 450. Actorloss = 0.1079690083861351, Critic_loss = 0.15055711567401886\n",
      "Episode 7. Training Carried out at step = 460. Actorloss = 0.1082209125161171, Critic_loss = 0.13961030542850494\n",
      "Episode 7. Training Carried out at step = 470. Actorloss = 0.10846077650785446, Critic_loss = 0.12751999497413635\n",
      "Episode 7. Training Carried out at step = 480. Actorloss = 0.10869394987821579, Critic_loss = 0.11485815048217773\n",
      "Episode 7. Training Carried out at step = 490. Actorloss = 0.10890242457389832, Critic_loss = 0.11222831904888153\n",
      "Episode 7. Training Carried out at step = 500. Actorloss = 0.1091088280081749, Critic_loss = 0.08929156512022018\n",
      "Episode 7. Training Carried out at step = 510. Actorloss = 0.10930480808019638, Critic_loss = 0.04663563147187233\n",
      "Episode 7. Training Carried out at step = 520. Actorloss = 0.10948818922042847, Critic_loss = 0.018295595422387123\n",
      "Episode 7. Training Carried out at step = 530. Actorloss = 0.10965479165315628, Critic_loss = 0.07192946970462799\n",
      "Episode 7. Training Carried out at step = 540. Actorloss = 0.10981307923793793, Critic_loss = 0.13555072247982025\n",
      "Episode 7. Training Carried out at step = 550. Actorloss = 0.10996577888727188, Critic_loss = 0.17364051938056946\n",
      "Episode 7. Training Carried out at step = 560. Actorloss = 0.11011546850204468, Critic_loss = 0.09046830236911774\n",
      "Episode 7. Training Carried out at step = 570. Actorloss = 0.11026433855295181, Critic_loss = 0.04034830629825592\n",
      "Episode 7. Training Carried out at step = 580. Actorloss = 0.11041413247585297, Critic_loss = 0.03348064795136452\n",
      "Episode 7. Training Carried out at step = 590. Actorloss = 0.11056645214557648, Critic_loss = 0.08952412009239197\n",
      "Episode 7. Training Carried out at step = 600. Actorloss = 0.11072275042533875, Critic_loss = 0.7026407718658447\n",
      "Episode 7. Training Carried out at step = 610. Actorloss = 0.11088471114635468, Critic_loss = 1.071319580078125\n",
      "Episode 7. Training Carried out at step = 620. Actorloss = 0.11105374991893768, Critic_loss = 1.4895579814910889\n",
      "Episode 7. Training Carried out at step = 630. Actorloss = 0.11123107373714447, Critic_loss = 1.5019124746322632\n",
      "Episode 7. Training Carried out at step = 640. Actorloss = 0.11141785979270935, Critic_loss = 1.489963173866272\n",
      "Episode 7. Training Carried out at step = 650. Actorloss = 0.11161509901285172, Critic_loss = 1.2081183195114136\n",
      "Episode 7. Training Carried out at step = 660. Actorloss = 0.11182382702827454, Critic_loss = 0.5807565450668335\n",
      "Episode 7. Training Carried out at step = 670. Actorloss = 0.11204488575458527, Critic_loss = 0.12842674553394318\n",
      "Episode 7. Training Carried out at step = 680. Actorloss = 0.11227905005216599, Critic_loss = 0.19282568991184235\n",
      "Episode 7. Training Carried out at step = 690. Actorloss = 0.11252297461032867, Critic_loss = 0.237931028008461\n",
      "Episode 7. Training Carried out at step = 700. Actorloss = 0.11277366429567337, Critic_loss = 0.22895075380802155\n",
      "Episode 7. Training Carried out at step = 710. Actorloss = 0.11302834749221802, Critic_loss = 0.19111308455467224\n",
      "Episode 7. Training Carried out at step = 720. Actorloss = 0.1132846474647522, Critic_loss = 0.1380937248468399\n",
      "Episode 7. Training Carried out at step = 730. Actorloss = 0.11354029923677444, Critic_loss = 0.1370554119348526\n",
      "Episode 7. Training Carried out at step = 740. Actorloss = 0.113793283700943, Critic_loss = 0.13383923470973969\n",
      "Episode 7. Training Carried out at step = 750. Actorloss = 0.11404183506965637, Critic_loss = 0.12812648713588715\n",
      "Episode 7. Training Carried out at step = 760. Actorloss = 0.11428437381982803, Critic_loss = 0.12086670100688934\n",
      "Episode 7. Training Carried out at step = 770. Actorloss = 0.11451936513185501, Critic_loss = 0.1123669296503067\n",
      "Episode 7. Training Carried out at step = 780. Actorloss = 0.1147455945611, Critic_loss = 0.102898009121418\n",
      "Episode 7. Training Carried out at step = 790. Actorloss = 0.1149618923664093, Critic_loss = 0.09281947463750839\n",
      "Episode 7. Training Carried out at step = 800. Actorloss = 0.11516717076301575, Critic_loss = 0.07429199665784836\n",
      "Episode 7. Training Carried out at step = 810. Actorloss = 0.11536058038473129, Critic_loss = 0.06424771994352341\n",
      "Episode 7. Training Carried out at step = 820. Actorloss = 0.11554137617349625, Critic_loss = 0.02756798081099987\n",
      "Episode 7. Training Carried out at step = 830. Actorloss = 0.11570923775434494, Critic_loss = 0.009459150955080986\n",
      "Episode 7. Training Carried out at step = 840. Actorloss = 0.11586587876081467, Critic_loss = 0.053123194724321365\n",
      "Episode 7. Training Carried out at step = 850. Actorloss = 0.11601443588733673, Critic_loss = 0.1276319921016693\n",
      "Episode 7. Training Carried out at step = 860. Actorloss = 0.11615756899118423, Critic_loss = 0.18743111193180084\n",
      "Episode 7. Training Carried out at step = 870. Actorloss = 0.11629780381917953, Critic_loss = 0.16048088669776917\n",
      "Episode 7. Training Carried out at step = 880. Actorloss = 0.11643730103969574, Critic_loss = 0.05607140064239502\n",
      "Episode 7. Training Carried out at step = 890. Actorloss = 0.11657799780368805, Critic_loss = 0.047627124935388565\n",
      "Episode 7. Training Carried out at step = 900. Actorloss = 0.11672162264585495, Critic_loss = 0.036102376878261566\n",
      "Episode 7. Training Carried out at step = 910. Actorloss = 0.11686939001083374, Critic_loss = 0.29536890983581543\n",
      "Episode 7. Training Carried out at step = 920. Actorloss = 0.11702263355255127, Critic_loss = 1.0710560083389282\n",
      "Episode 7. Training Carried out at step = 930. Actorloss = 0.11718278378248215, Critic_loss = 1.402948260307312\n",
      "Episode 7. Training Carried out at step = 940. Actorloss = 0.11735118925571442, Critic_loss = 1.6165080070495605\n",
      "Episode 7. Training Carried out at step = 950. Actorloss = 0.11752885580062866, Critic_loss = 1.679162621498108\n",
      "Episode 7. Training Carried out at step = 960. Actorloss = 0.11771684139966965, Critic_loss = 1.466652274131775\n",
      "Episode 7. Training Carried out at step = 970. Actorloss = 0.11791608482599258, Critic_loss = 0.8808833360671997\n",
      "Episode 7. Training Carried out at step = 980. Actorloss = 0.11812744289636612, Critic_loss = 0.2640511393547058\n",
      "Episode 7. Training Carried out at step = 990. Actorloss = 0.11835172772407532, Critic_loss = 0.11953931301832199\n",
      "Episode 7. Training Carried out at step = 1000. Actorloss = 0.11858560144901276, Critic_loss = 0.2203953117132187\n",
      "Episode 8. Training Carried out at step = 10. Actorloss = 0.11640762537717819, Critic_loss = 4.355868339538574\n",
      "Episode 8. Training Carried out at step = 20. Actorloss = 0.11700546741485596, Critic_loss = 17.430723190307617\n",
      "Episode 8. Training Carried out at step = 30. Actorloss = 0.11734544485807419, Critic_loss = 22.697078704833984\n",
      "Episode 8. Training Carried out at step = 40. Actorloss = 0.11774155497550964, Critic_loss = 3.811060905456543\n",
      "Episode 8. Training Carried out at step = 50. Actorloss = 0.11801104247570038, Critic_loss = 0.3062891364097595\n",
      "Episode 8. Training Carried out at step = 60. Actorloss = 0.11840633302927017, Critic_loss = 0.32504379749298096\n",
      "Episode 8. Training Carried out at step = 70. Actorloss = 0.11882025003433228, Critic_loss = 0.3124593496322632\n",
      "Episode 8. Training Carried out at step = 80. Actorloss = 0.11923426389694214, Critic_loss = 0.26164841651916504\n",
      "Episode 8. Training Carried out at step = 90. Actorloss = 0.11970492452383041, Critic_loss = 0.21185137331485748\n",
      "Episode 8. Training Carried out at step = 100. Actorloss = 0.12012430280447006, Critic_loss = 0.20876425504684448\n",
      "Episode 8. Training Carried out at step = 110. Actorloss = 0.12054004520177841, Critic_loss = 0.20237202942371368\n",
      "Episode 8. Training Carried out at step = 120. Actorloss = 0.12095005065202713, Critic_loss = 0.19388416409492493\n",
      "Episode 8. Training Carried out at step = 130. Actorloss = 0.1213526651263237, Critic_loss = 0.18303759396076202\n",
      "Episode 8. Training Carried out at step = 140. Actorloss = 0.1217789426445961, Critic_loss = 0.1704789102077484\n",
      "Episode 8. Training Carried out at step = 150. Actorloss = 0.12222064286470413, Critic_loss = 0.156765416264534\n",
      "Episode 8. Training Carried out at step = 160. Actorloss = 0.12251335382461548, Critic_loss = 0.14279678463935852\n",
      "Episode 8. Training Carried out at step = 170. Actorloss = 0.12269219011068344, Critic_loss = 0.13302822411060333\n",
      "Episode 8. Training Carried out at step = 180. Actorloss = 0.1228361651301384, Critic_loss = 0.12023109942674637\n",
      "Episode 8. Training Carried out at step = 190. Actorloss = 0.12310735881328583, Critic_loss = 0.07427836954593658\n",
      "Episode 8. Training Carried out at step = 200. Actorloss = 0.12345285713672638, Critic_loss = 0.01228378713130951\n",
      "Episode 8. Training Carried out at step = 210. Actorloss = 0.12374883890151978, Critic_loss = 0.03997577726840973\n",
      "Episode 8. Training Carried out at step = 220. Actorloss = 0.12398919463157654, Critic_loss = 0.09510816633701324\n",
      "Episode 8. Training Carried out at step = 230. Actorloss = 0.12415891885757446, Critic_loss = 0.14547385275363922\n",
      "Episode 8. Training Carried out at step = 240. Actorloss = 0.12434899806976318, Critic_loss = 0.13243363797664642\n",
      "Episode 8. Training Carried out at step = 250. Actorloss = 0.12471728771924973, Critic_loss = 0.03186094015836716\n",
      "Episode 8. Training Carried out at step = 260. Actorloss = 0.12489593029022217, Critic_loss = 0.02484147809445858\n",
      "Episode 8. Training Carried out at step = 270. Actorloss = 0.12504275143146515, Critic_loss = 0.012666167691349983\n",
      "Episode 8. Training Carried out at step = 280. Actorloss = 0.12519164383411407, Critic_loss = 0.24912941455841064\n",
      "Episode 8. Training Carried out at step = 290. Actorloss = 0.1254023015499115, Critic_loss = 1.0383179187774658\n",
      "Episode 8. Training Carried out at step = 300. Actorloss = 0.12558361887931824, Critic_loss = 1.3675262928009033\n",
      "Episode 8. Training Carried out at step = 310. Actorloss = 0.12576846778392792, Critic_loss = 1.5742971897125244\n",
      "Episode 8. Training Carried out at step = 320. Actorloss = 0.12596824765205383, Critic_loss = 1.4991012811660767\n",
      "Episode 8. Training Carried out at step = 330. Actorloss = 0.12617354094982147, Critic_loss = 1.3570905923843384\n",
      "Episode 8. Training Carried out at step = 340. Actorloss = 0.12638646364212036, Critic_loss = 0.8199518918991089\n",
      "Episode 8. Training Carried out at step = 350. Actorloss = 0.1266094595193863, Critic_loss = 0.2668050527572632\n",
      "Episode 8. Training Carried out at step = 360. Actorloss = 0.12684325873851776, Critic_loss = 0.16029655933380127\n",
      "Episode 8. Training Carried out at step = 370. Actorloss = 0.12708646059036255, Critic_loss = 0.2601524591445923\n",
      "Episode 8. Training Carried out at step = 380. Actorloss = 0.12733563780784607, Critic_loss = 0.2501225173473358\n",
      "Episode 8. Training Carried out at step = 390. Actorloss = 0.12758778035640717, Critic_loss = 0.22917883098125458\n",
      "Episode 8. Training Carried out at step = 400. Actorloss = 0.12784074246883392, Critic_loss = 0.15650632977485657\n",
      "Episode 8. Training Carried out at step = 410. Actorloss = 0.12809251248836517, Critic_loss = 0.14193826913833618\n",
      "Episode 8. Training Carried out at step = 420. Actorloss = 0.12834125757217407, Critic_loss = 0.13804389536380768\n",
      "Episode 8. Training Carried out at step = 430. Actorloss = 0.12858521938323975, Critic_loss = 0.13210232555866241\n",
      "Episode 8. Training Carried out at step = 440. Actorloss = 0.12882287800312042, Critic_loss = 0.12469148635864258\n",
      "Episode 8. Training Carried out at step = 450. Actorloss = 0.12905292212963104, Critic_loss = 0.11584634333848953\n",
      "Episode 8. Training Carried out at step = 460. Actorloss = 0.12927405536174774, Critic_loss = 0.10644343495368958\n",
      "Episode 8. Training Carried out at step = 470. Actorloss = 0.12948521971702576, Critic_loss = 0.09627615660429001\n",
      "Episode 8. Training Carried out at step = 480. Actorloss = 0.1296853870153427, Critic_loss = 0.08462151139974594\n",
      "Episode 8. Training Carried out at step = 490. Actorloss = 0.12987373769283295, Critic_loss = 0.07887999713420868\n",
      "Episode 8. Training Carried out at step = 500. Actorloss = 0.13004952669143677, Critic_loss = 0.06138099357485771\n",
      "Episode 8. Training Carried out at step = 510. Actorloss = 0.13021206855773926, Critic_loss = 0.02584201842546463\n",
      "Episode 8. Training Carried out at step = 520. Actorloss = 0.13036131858825684, Critic_loss = 0.030881958082318306\n",
      "Episode 8. Training Carried out at step = 530. Actorloss = 0.13050000369548798, Critic_loss = 0.09538739919662476\n",
      "Episode 8. Training Carried out at step = 540. Actorloss = 0.13063108921051025, Critic_loss = 0.17203934490680695\n",
      "Episode 8. Training Carried out at step = 550. Actorloss = 0.13075710833072662, Critic_loss = 0.21208393573760986\n",
      "Episode 8. Training Carried out at step = 560. Actorloss = 0.1308804452419281, Critic_loss = 0.07761618494987488\n",
      "Episode 8. Training Carried out at step = 570. Actorloss = 0.1310032308101654, Critic_loss = 0.053630370646715164\n",
      "Episode 8. Training Carried out at step = 580. Actorloss = 0.13112734258174896, Critic_loss = 0.04564031958580017\n",
      "Episode 8. Training Carried out at step = 590. Actorloss = 0.1312544047832489, Critic_loss = 0.042695432901382446\n",
      "Episode 8. Training Carried out at step = 600. Actorloss = 0.13138531148433685, Critic_loss = 0.8200854063034058\n",
      "Episode 8. Training Carried out at step = 610. Actorloss = 0.1315215677022934, Critic_loss = 1.4319592714309692\n",
      "Episode 8. Training Carried out at step = 620. Actorloss = 0.13166452944278717, Critic_loss = 1.6534380912780762\n",
      "Episode 8. Training Carried out at step = 630. Actorloss = 0.13181541860103607, Critic_loss = 1.6710007190704346\n",
      "Episode 8. Training Carried out at step = 640. Actorloss = 0.13197526335716248, Critic_loss = 1.622530221939087\n",
      "Episode 8. Training Carried out at step = 650. Actorloss = 0.13214513659477234, Critic_loss = 1.1910715103149414\n",
      "Episode 8. Training Carried out at step = 660. Actorloss = 0.13232585787773132, Critic_loss = 0.5144962072372437\n",
      "Episode 8. Training Carried out at step = 670. Actorloss = 0.13251833617687225, Critic_loss = 0.10958315432071686\n",
      "Episode 8. Training Carried out at step = 680. Actorloss = 0.13272185623645782, Critic_loss = 0.2033298760652542\n",
      "Episode 8. Training Carried out at step = 690. Actorloss = 0.1329333633184433, Critic_loss = 0.20381838083267212\n",
      "Episode 8. Training Carried out at step = 700. Actorloss = 0.1331501603126526, Critic_loss = 0.19294622540473938\n",
      "Episode 8. Training Carried out at step = 710. Actorloss = 0.13336971402168274, Critic_loss = 0.15649619698524475\n",
      "Episode 8. Training Carried out at step = 720. Actorloss = 0.13358984887599945, Critic_loss = 0.10796021670103073\n",
      "Episode 8. Training Carried out at step = 730. Actorloss = 0.13380852341651917, Critic_loss = 0.10644348710775375\n",
      "Episode 8. Training Carried out at step = 740. Actorloss = 0.134023979306221, Critic_loss = 0.1030738577246666\n",
      "Episode 8. Training Carried out at step = 750. Actorloss = 0.13423451781272888, Critic_loss = 0.09817639738321304\n",
      "Episode 8. Training Carried out at step = 760. Actorloss = 0.13443870842456818, Critic_loss = 0.09207118302583694\n",
      "Episode 8. Training Carried out at step = 770. Actorloss = 0.13463525474071503, Critic_loss = 0.08507849276065826\n",
      "Episode 8. Training Carried out at step = 780. Actorloss = 0.1348230242729187, Critic_loss = 0.07737451791763306\n",
      "Episode 8. Training Carried out at step = 790. Actorloss = 0.13500092923641205, Critic_loss = 0.0695512443780899\n",
      "Episode 8. Training Carried out at step = 800. Actorloss = 0.13516803085803986, Critic_loss = 0.06449224799871445\n",
      "Episode 8. Training Carried out at step = 810. Actorloss = 0.13532353937625885, Critic_loss = 0.05275369808077812\n",
      "Episode 8. Training Carried out at step = 820. Actorloss = 0.1354667693376541, Critic_loss = 0.028539732098579407\n",
      "Episode 8. Training Carried out at step = 830. Actorloss = 0.13559755682945251, Critic_loss = 0.013106550090014935\n",
      "Episode 8. Training Carried out at step = 840. Actorloss = 0.13571761548519135, Critic_loss = 0.06524605304002762\n",
      "Episode 8. Training Carried out at step = 850. Actorloss = 0.1358298510313034, Critic_loss = 0.15240831673145294\n",
      "Episode 8. Training Carried out at step = 860. Actorloss = 0.13593687117099762, Critic_loss = 0.21842020750045776\n",
      "Episode 8. Training Carried out at step = 870. Actorloss = 0.13604100048542023, Critic_loss = 0.15199589729309082\n",
      "Episode 8. Training Carried out at step = 880. Actorloss = 0.13614438474178314, Critic_loss = 0.06915273517370224\n",
      "Episode 8. Training Carried out at step = 890. Actorloss = 0.13624876737594604, Critic_loss = 0.05923968926072121\n",
      "Episode 8. Training Carried out at step = 900. Actorloss = 0.13635605573654175, Critic_loss = 0.05119231715798378\n",
      "Episode 8. Training Carried out at step = 910. Actorloss = 0.1364675909280777, Critic_loss = 0.21985770761966705\n",
      "Episode 8. Training Carried out at step = 920. Actorloss = 0.1365845501422882, Critic_loss = 1.3217298984527588\n",
      "Episode 8. Training Carried out at step = 930. Actorloss = 0.13670840859413147, Critic_loss = 1.549332857131958\n",
      "Episode 8. Training Carried out at step = 940. Actorloss = 0.13684020936489105, Critic_loss = 1.7434051036834717\n",
      "Episode 8. Training Carried out at step = 950. Actorloss = 0.1369810402393341, Critic_loss = 1.8145346641540527\n",
      "Episode 8. Training Carried out at step = 960. Actorloss = 0.13713181018829346, Critic_loss = 1.4963386058807373\n",
      "Episode 8. Training Carried out at step = 970. Actorloss = 0.13729342818260193, Critic_loss = 0.8099972605705261\n",
      "Episode 8. Training Carried out at step = 980. Actorloss = 0.13746662437915802, Critic_loss = 0.18326514959335327\n",
      "Episode 8. Training Carried out at step = 990. Actorloss = 0.1376521736383438, Critic_loss = 0.1534464806318283\n",
      "Episode 8. Training Carried out at step = 1000. Actorloss = 0.1378469169139862, Critic_loss = 0.19549357891082764\n",
      "Episode 9. Training Carried out at step = 10. Actorloss = 0.1367070972919464, Critic_loss = 4.1309895515441895\n",
      "Episode 9. Training Carried out at step = 20. Actorloss = 0.13704650104045868, Critic_loss = 16.780717849731445\n",
      "Episode 9. Training Carried out at step = 30. Actorloss = 0.13753047585487366, Critic_loss = 20.106969833374023\n",
      "Episode 9. Training Carried out at step = 40. Actorloss = 0.13776054978370667, Critic_loss = 2.76181960105896\n",
      "Episode 9. Training Carried out at step = 50. Actorloss = 0.13805584609508514, Critic_loss = 0.7817193269729614\n",
      "Episode 9. Training Carried out at step = 60. Actorloss = 0.13839128613471985, Critic_loss = 1.1528372764587402\n",
      "Episode 9. Training Carried out at step = 70. Actorloss = 0.1387307494878769, Critic_loss = 0.8953056931495667\n",
      "Episode 9. Training Carried out at step = 80. Actorloss = 0.13907353579998016, Critic_loss = 0.4691694378852844\n",
      "Episode 9. Training Carried out at step = 90. Actorloss = 0.13941141963005066, Critic_loss = 0.2023516595363617\n",
      "Episode 9. Training Carried out at step = 100. Actorloss = 0.1397222876548767, Critic_loss = 0.13661715388298035\n",
      "Episode 9. Training Carried out at step = 110. Actorloss = 0.14002478122711182, Critic_loss = 0.11566489189863205\n",
      "Episode 9. Training Carried out at step = 120. Actorloss = 0.14032092690467834, Critic_loss = 0.08040642738342285\n",
      "Episode 9. Training Carried out at step = 130. Actorloss = 0.14061282575130463, Critic_loss = 0.038169633597135544\n",
      "Episode 9. Training Carried out at step = 140. Actorloss = 0.1409422755241394, Critic_loss = 0.009720723144710064\n",
      "Episode 9. Training Carried out at step = 150. Actorloss = 0.14124740660190582, Critic_loss = 0.0013468849938362837\n",
      "Episode 9. Training Carried out at step = 160. Actorloss = 0.1414984166622162, Critic_loss = 0.007400325499475002\n",
      "Episode 9. Training Carried out at step = 170. Actorloss = 0.14164729416370392, Critic_loss = 0.015210675075650215\n",
      "Episode 9. Training Carried out at step = 180. Actorloss = 0.14179390668869019, Critic_loss = 0.013694098219275475\n",
      "Episode 9. Training Carried out at step = 190. Actorloss = 0.14206276834011078, Critic_loss = 0.023882916197180748\n",
      "Episode 9. Training Carried out at step = 200. Actorloss = 0.14233995974063873, Critic_loss = 0.1609288603067398\n",
      "Episode 9. Training Carried out at step = 210. Actorloss = 0.14259809255599976, Critic_loss = 0.3366529941558838\n",
      "Episode 9. Training Carried out at step = 220. Actorloss = 0.14281117916107178, Critic_loss = 0.454757422208786\n",
      "Episode 9. Training Carried out at step = 230. Actorloss = 0.1430193930864334, Critic_loss = 0.5174787640571594\n",
      "Episode 9. Training Carried out at step = 240. Actorloss = 0.14321458339691162, Critic_loss = 0.472807914018631\n",
      "Episode 9. Training Carried out at step = 250. Actorloss = 0.1434217393398285, Critic_loss = 0.2541647255420685\n",
      "Episode 9. Training Carried out at step = 260. Actorloss = 0.14361265301704407, Critic_loss = 0.2281835526227951\n",
      "Episode 9. Training Carried out at step = 270. Actorloss = 0.14379510283470154, Critic_loss = 0.17817039787769318\n",
      "Episode 9. Training Carried out at step = 280. Actorloss = 0.14396338164806366, Critic_loss = 0.8970339894294739\n",
      "Episode 9. Training Carried out at step = 290. Actorloss = 0.14415691792964935, Critic_loss = 1.3897604942321777\n",
      "Episode 9. Training Carried out at step = 300. Actorloss = 0.14431920647621155, Critic_loss = 2.011014699935913\n",
      "Episode 9. Training Carried out at step = 310. Actorloss = 0.14451418817043304, Critic_loss = 2.268141269683838\n",
      "Episode 9. Training Carried out at step = 320. Actorloss = 0.14470243453979492, Critic_loss = 2.3601555824279785\n",
      "Episode 9. Training Carried out at step = 330. Actorloss = 0.1448861062526703, Critic_loss = 2.1707003116607666\n",
      "Episode 9. Training Carried out at step = 340. Actorloss = 0.1450682431459427, Critic_loss = 1.5154138803482056\n",
      "Episode 9. Training Carried out at step = 350. Actorloss = 0.14524786174297333, Critic_loss = 0.7390905618667603\n",
      "Episode 9. Training Carried out at step = 360. Actorloss = 0.1454218626022339, Critic_loss = 0.11311119794845581\n",
      "Episode 9. Training Carried out at step = 370. Actorloss = 0.14559325575828552, Critic_loss = 0.03507505729794502\n",
      "Episode 9. Training Carried out at step = 380. Actorloss = 0.14578069746494293, Critic_loss = 0.03639634698629379\n",
      "Episode 9. Training Carried out at step = 390. Actorloss = 0.14596162736415863, Critic_loss = 0.033578913658857346\n",
      "Episode 9. Training Carried out at step = 400. Actorloss = 0.1461591124534607, Critic_loss = 0.014219745993614197\n",
      "Episode 9. Training Carried out at step = 410. Actorloss = 0.1463436633348465, Critic_loss = 0.01099010743200779\n",
      "Episode 9. Training Carried out at step = 420. Actorloss = 0.14652147889137268, Critic_loss = 0.011622980237007141\n",
      "Episode 9. Training Carried out at step = 430. Actorloss = 0.14667735993862152, Critic_loss = 0.011993741616606712\n",
      "Episode 9. Training Carried out at step = 440. Actorloss = 0.1468757539987564, Critic_loss = 0.011226659640669823\n",
      "Episode 9. Training Carried out at step = 450. Actorloss = 0.1470504105091095, Critic_loss = 0.01041730772703886\n",
      "Episode 9. Training Carried out at step = 460. Actorloss = 0.14722347259521484, Critic_loss = 0.009303078055381775\n",
      "Episode 9. Training Carried out at step = 470. Actorloss = 0.14739888906478882, Critic_loss = 0.008111827075481415\n",
      "Episode 9. Training Carried out at step = 480. Actorloss = 0.14756116271018982, Critic_loss = 0.006418473552912474\n",
      "Episode 9. Training Carried out at step = 490. Actorloss = 0.14769180119037628, Critic_loss = 0.007819801568984985\n",
      "Episode 9. Training Carried out at step = 500. Actorloss = 0.14784885942935944, Critic_loss = 0.00460453424602747\n",
      "Episode 9. Training Carried out at step = 510. Actorloss = 0.1480238288640976, Critic_loss = 0.00965132936835289\n",
      "Episode 9. Training Carried out at step = 520. Actorloss = 0.14837364852428436, Critic_loss = 0.1511204093694687\n",
      "Episode 9. Training Carried out at step = 530. Actorloss = 0.14851580560207367, Critic_loss = 0.26032888889312744\n",
      "Episode 9. Training Carried out at step = 540. Actorloss = 0.148691788315773, Critic_loss = 0.3573911190032959\n",
      "Episode 9. Training Carried out at step = 550. Actorloss = 0.14886102080345154, Critic_loss = 0.40158897638320923\n",
      "Episode 9. Training Carried out at step = 560. Actorloss = 0.14904342591762543, Critic_loss = 0.21027226746082306\n",
      "Episode 9. Training Carried out at step = 570. Actorloss = 0.1492455005645752, Critic_loss = 0.1479727178812027\n",
      "Episode 9. Training Carried out at step = 580. Actorloss = 0.14942297339439392, Critic_loss = 0.12479040771722794\n",
      "Episode 9. Training Carried out at step = 590. Actorloss = 0.1494360864162445, Critic_loss = 0.13815253973007202\n",
      "Episode 9. Training Carried out at step = 600. Actorloss = 0.14951080083847046, Critic_loss = 0.9497079253196716\n",
      "Episode 9. Training Carried out at step = 610. Actorloss = 0.14968761801719666, Critic_loss = 1.2842366695404053\n",
      "Episode 9. Training Carried out at step = 620. Actorloss = 0.14981518685817719, Critic_loss = 1.6630315780639648\n",
      "Episode 9. Training Carried out at step = 630. Actorloss = 0.14987970888614655, Critic_loss = 1.6325268745422363\n",
      "Episode 9. Training Carried out at step = 640. Actorloss = 0.15006282925605774, Critic_loss = 1.4590402841567993\n",
      "Episode 9. Training Carried out at step = 650. Actorloss = 0.15018154680728912, Critic_loss = 0.9878597855567932\n",
      "Episode 9. Training Carried out at step = 660. Actorloss = 0.15029096603393555, Critic_loss = 0.32683613896369934\n",
      "Episode 9. Training Carried out at step = 670. Actorloss = 0.15038470923900604, Critic_loss = 0.0957569107413292\n",
      "Episode 9. Training Carried out at step = 680. Actorloss = 0.15053746104240417, Critic_loss = 0.527679443359375\n",
      "Episode 9. Training Carried out at step = 690. Actorloss = 0.15063421428203583, Critic_loss = 0.6070570349693298\n",
      "Episode 9. Training Carried out at step = 700. Actorloss = 0.15073321759700775, Critic_loss = 0.6283778548240662\n",
      "Episode 9. Training Carried out at step = 710. Actorloss = 0.1508338302373886, Critic_loss = 0.5898470878601074\n",
      "Episode 9. Training Carried out at step = 720. Actorloss = 0.15093660354614258, Critic_loss = 0.4847519099712372\n",
      "Episode 9. Training Carried out at step = 730. Actorloss = 0.15103323757648468, Critic_loss = 0.4618222713470459\n",
      "Episode 9. Training Carried out at step = 740. Actorloss = 0.15106956660747528, Critic_loss = 0.42866793274879456\n",
      "Episode 9. Training Carried out at step = 750. Actorloss = 0.15118305385112762, Critic_loss = 0.3790578544139862\n",
      "Episode 9. Training Carried out at step = 760. Actorloss = 0.15126101672649384, Critic_loss = 0.3276856541633606\n",
      "Episode 9. Training Carried out at step = 770. Actorloss = 0.15131862461566925, Critic_loss = 0.27593618631362915\n",
      "Episode 9. Training Carried out at step = 780. Actorloss = 0.1513882577419281, Critic_loss = 0.22509922087192535\n",
      "Episode 9. Training Carried out at step = 790. Actorloss = 0.15144912898540497, Critic_loss = 0.17921333014965057\n",
      "Episode 9. Training Carried out at step = 800. Actorloss = 0.15148787200450897, Critic_loss = 0.15003156661987305\n",
      "Episode 9. Training Carried out at step = 810. Actorloss = 0.15151678025722504, Critic_loss = 0.11127065867185593\n",
      "Episode 9. Training Carried out at step = 820. Actorloss = 0.1516253799200058, Critic_loss = 0.041401833295822144\n",
      "Episode 9. Training Carried out at step = 830. Actorloss = 0.1517951339483261, Critic_loss = 0.013832597061991692\n",
      "Episode 9. Training Carried out at step = 840. Actorloss = 0.1520235389471054, Critic_loss = 0.07173296064138412\n",
      "Episode 9. Training Carried out at step = 850. Actorloss = 0.152055561542511, Critic_loss = 0.16714392602443695\n",
      "Episode 9. Training Carried out at step = 860. Actorloss = 0.1521017849445343, Critic_loss = 0.24535496532917023\n",
      "Episode 9. Training Carried out at step = 870. Actorloss = 0.15211106836795807, Critic_loss = 0.1794135868549347\n",
      "Episode 9. Training Carried out at step = 880. Actorloss = 0.15214726328849792, Critic_loss = 0.09264220297336578\n",
      "Episode 9. Training Carried out at step = 890. Actorloss = 0.15219931304454803, Critic_loss = 0.07883547991514206\n",
      "Episode 9. Training Carried out at step = 900. Actorloss = 0.15237225592136383, Critic_loss = 0.06898334622383118\n",
      "Episode 9. Training Carried out at step = 910. Actorloss = 0.152142733335495, Critic_loss = 0.23709149658679962\n",
      "Episode 9. Training Carried out at step = 920. Actorloss = 0.15209485590457916, Critic_loss = 1.3030887842178345\n",
      "Episode 9. Training Carried out at step = 930. Actorloss = 0.15205761790275574, Critic_loss = 1.5174604654312134\n",
      "Episode 9. Training Carried out at step = 940. Actorloss = 0.15201632678508759, Critic_loss = 1.6998112201690674\n",
      "Episode 9. Training Carried out at step = 950. Actorloss = 0.15205179154872894, Critic_loss = 1.7646523714065552\n",
      "Episode 9. Training Carried out at step = 960. Actorloss = 0.15199114382266998, Critic_loss = 1.395889401435852\n",
      "Episode 9. Training Carried out at step = 970. Actorloss = 0.15196679532527924, Critic_loss = 0.7268031239509583\n",
      "Episode 9. Training Carried out at step = 980. Actorloss = 0.1519349068403244, Critic_loss = 0.15434645116329193\n",
      "Episode 9. Training Carried out at step = 990. Actorloss = 0.1520928591489792, Critic_loss = 0.2249513566493988\n",
      "Episode 9. Training Carried out at step = 1000. Actorloss = 0.15204815566539764, Critic_loss = 0.3223172426223755\n",
      "Episode 9. Model checkpoint is saved \n",
      "\n",
      "Episode 9. tf.summary -> Tensorboard is updated \n",
      "\n",
      "Episode 10. Training Carried out at step = 10. Actorloss = 0.14830853044986725, Critic_loss = 4.13961935043335\n",
      "Episode 10. Training Carried out at step = 20. Actorloss = 0.15135054290294647, Critic_loss = 16.777507781982422\n",
      "Episode 10. Training Carried out at step = 30. Actorloss = 0.14922462403774261, Critic_loss = 21.49920082092285\n",
      "Episode 10. Training Carried out at step = 40. Actorloss = 0.15215212106704712, Critic_loss = 3.910874843597412\n",
      "Episode 10. Training Carried out at step = 50. Actorloss = 0.15234000980854034, Critic_loss = 0.3299693167209625\n",
      "Episode 10. Training Carried out at step = 60. Actorloss = 0.15245027840137482, Critic_loss = 0.3697832524776459\n",
      "Episode 10. Training Carried out at step = 70. Actorloss = 0.15256468951702118, Critic_loss = 0.35686197876930237\n",
      "Episode 10. Training Carried out at step = 80. Actorloss = 0.1525840163230896, Critic_loss = 0.30063381791114807\n",
      "Episode 10. Training Carried out at step = 90. Actorloss = 0.15261168777942657, Critic_loss = 0.24882709980010986\n",
      "Episode 10. Training Carried out at step = 100. Actorloss = 0.15263628959655762, Critic_loss = 0.24035562574863434\n",
      "Episode 10. Training Carried out at step = 110. Actorloss = 0.1526450663805008, Critic_loss = 0.21928055584430695\n",
      "Episode 10. Training Carried out at step = 120. Actorloss = 0.1526428908109665, Critic_loss = 0.17962388694286346\n",
      "Episode 10. Training Carried out at step = 130. Actorloss = 0.15262971818447113, Critic_loss = 0.12191648036241531\n",
      "Episode 10. Training Carried out at step = 140. Actorloss = 0.15332470834255219, Critic_loss = 0.05018178001046181\n",
      "Episode 10. Training Carried out at step = 150. Actorloss = 0.1545657217502594, Critic_loss = 0.006005465053021908\n",
      "Episode 10. Training Carried out at step = 160. Actorloss = 0.15452738106250763, Critic_loss = 0.02145996317267418\n",
      "Episode 10. Training Carried out at step = 170. Actorloss = 0.1543782502412796, Critic_loss = 0.06545322388410568\n",
      "Episode 10. Training Carried out at step = 180. Actorloss = 0.15448203682899475, Critic_loss = 0.08107636123895645\n",
      "Episode 10. Training Carried out at step = 190. Actorloss = 0.15438243746757507, Critic_loss = 0.09601372480392456\n",
      "Episode 10. Training Carried out at step = 200. Actorloss = 0.15431368350982666, Critic_loss = 0.22648262977600098\n",
      "Episode 10. Training Carried out at step = 210. Actorloss = 0.15404096245765686, Critic_loss = 0.306179940700531\n",
      "Episode 10. Training Carried out at step = 220. Actorloss = 0.15418396890163422, Critic_loss = 0.28230515122413635\n",
      "Episode 10. Training Carried out at step = 230. Actorloss = 0.1540641486644745, Critic_loss = 0.22703047096729279\n",
      "Episode 10. Training Carried out at step = 240. Actorloss = 0.15399602055549622, Critic_loss = 0.13038410246372223\n",
      "Episode 10. Training Carried out at step = 250. Actorloss = 0.15420520305633545, Critic_loss = 0.00816996768116951\n",
      "Episode 10. Training Carried out at step = 260. Actorloss = 0.1540939211845398, Critic_loss = 0.004142360761761665\n",
      "Episode 10. Training Carried out at step = 270. Actorloss = 0.15389466285705566, Critic_loss = 0.03277925029397011\n",
      "Episode 10. Training Carried out at step = 280. Actorloss = 0.1535787284374237, Critic_loss = 0.11310377717018127\n",
      "Episode 10. Training Carried out at step = 290. Actorloss = 0.15318416059017181, Critic_loss = 0.5928177833557129\n",
      "Episode 10. Training Carried out at step = 300. Actorloss = 0.15293820202350616, Critic_loss = 0.7865275740623474\n",
      "Episode 10. Training Carried out at step = 310. Actorloss = 0.1529761701822281, Critic_loss = 0.8538659811019897\n",
      "Episode 10. Training Carried out at step = 320. Actorloss = 0.15293486416339874, Critic_loss = 0.6516808271408081\n",
      "Episode 10. Training Carried out at step = 330. Actorloss = 0.1527029275894165, Critic_loss = 0.3865007162094116\n",
      "Episode 10. Training Carried out at step = 340. Actorloss = 0.15251876413822174, Critic_loss = 0.03770913928747177\n",
      "Episode 10. Training Carried out at step = 350. Actorloss = 0.1524631381034851, Critic_loss = 0.2501749098300934\n",
      "Episode 10. Training Carried out at step = 360. Actorloss = 0.15281078219413757, Critic_loss = 1.956351399421692\n",
      "Episode 10. Training Carried out at step = 370. Actorloss = 0.15279355645179749, Critic_loss = 2.820953369140625\n",
      "Episode 10. Training Carried out at step = 380. Actorloss = 0.15271960198879242, Critic_loss = 2.782538414001465\n",
      "Episode 10. Training Carried out at step = 390. Actorloss = 0.15250857174396515, Critic_loss = 2.5097854137420654\n",
      "Episode 10. Training Carried out at step = 400. Actorloss = 0.15271759033203125, Critic_loss = 1.830763339996338\n",
      "Episode 10. Training Carried out at step = 410. Actorloss = 0.15287944674491882, Critic_loss = 1.3830580711364746\n",
      "Episode 10. Training Carried out at step = 420. Actorloss = 0.15276089310646057, Critic_loss = 1.0228558778762817\n",
      "Episode 10. Training Carried out at step = 430. Actorloss = 0.15284651517868042, Critic_loss = 0.6696905493736267\n",
      "Episode 10. Training Carried out at step = 440. Actorloss = 0.1528742015361786, Critic_loss = 0.43772897124290466\n",
      "Episode 10. Training Carried out at step = 450. Actorloss = 0.1528473049402237, Critic_loss = 0.2780481278896332\n",
      "Episode 10. Training Carried out at step = 460. Actorloss = 0.1527094691991806, Critic_loss = 0.1695551872253418\n",
      "Episode 10. Training Carried out at step = 470. Actorloss = 0.15255172550678253, Critic_loss = 0.09544161707162857\n",
      "Episode 10. Training Carried out at step = 480. Actorloss = 0.1527709811925888, Critic_loss = 0.06974324584007263\n",
      "Episode 10. Training Carried out at step = 490. Actorloss = 0.15319758653640747, Critic_loss = 0.132978618144989\n",
      "Episode 10. Training Carried out at step = 500. Actorloss = 0.15247178077697754, Critic_loss = 0.003452805569395423\n",
      "Episode 10. Training Carried out at step = 510. Actorloss = 0.1521342545747757, Critic_loss = 0.04706595838069916\n",
      "Episode 10. Training Carried out at step = 520. Actorloss = 0.1514594852924347, Critic_loss = 0.22942028939723969\n",
      "Episode 10. Training Carried out at step = 530. Actorloss = 0.15145979821681976, Critic_loss = 0.274457722902298\n",
      "Episode 10. Training Carried out at step = 540. Actorloss = 0.15116624534130096, Critic_loss = 0.21606123447418213\n",
      "Episode 10. Training Carried out at step = 550. Actorloss = 0.15073563158512115, Critic_loss = 0.06772027164697647\n",
      "Episode 10. Training Carried out at step = 560. Actorloss = 0.15089695155620575, Critic_loss = 0.12096123397350311\n",
      "Episode 10. Training Carried out at step = 570. Actorloss = 0.1507464200258255, Critic_loss = 0.056477900594472885\n",
      "Episode 10. Training Carried out at step = 580. Actorloss = 0.15073028206825256, Critic_loss = 0.045882705599069595\n",
      "Episode 10. Training Carried out at step = 590. Actorloss = 0.15165264904499054, Critic_loss = 0.02598501741886139\n",
      "Episode 10. Training Carried out at step = 600. Actorloss = 0.15220089256763458, Critic_loss = 0.029548492282629013\n",
      "Episode 10. Training Carried out at step = 610. Actorloss = 0.1521519422531128, Critic_loss = 0.1974145621061325\n",
      "Episode 10. Training Carried out at step = 620. Actorloss = 0.15125876665115356, Critic_loss = 0.030329504981637\n",
      "Episode 10. Training Carried out at step = 630. Actorloss = 0.15056659281253815, Critic_loss = 0.0914044976234436\n",
      "Episode 10. Training Carried out at step = 640. Actorloss = 0.15042531490325928, Critic_loss = 0.5435886383056641\n",
      "Episode 10. Training Carried out at step = 650. Actorloss = 0.14996469020843506, Critic_loss = 1.3704164028167725\n",
      "Episode 10. Training Carried out at step = 660. Actorloss = 0.14937905967235565, Critic_loss = 1.8900909423828125\n",
      "Episode 10. Training Carried out at step = 670. Actorloss = 0.14855678379535675, Critic_loss = 2.0516014099121094\n",
      "Episode 10. Training Carried out at step = 680. Actorloss = 0.14714348316192627, Critic_loss = 2.1300692558288574\n",
      "Episode 10. Training Carried out at step = 690. Actorloss = 0.14742590487003326, Critic_loss = 0.6206222176551819\n",
      "Episode 10. Training Carried out at step = 700. Actorloss = 0.14950990676879883, Critic_loss = 0.30594268441200256\n",
      "Episode 10. Training Carried out at step = 710. Actorloss = 0.14764808118343353, Critic_loss = 0.060640111565589905\n",
      "Episode 10. Training Carried out at step = 720. Actorloss = 0.14780478179454803, Critic_loss = 0.09334409981966019\n",
      "Episode 10. Training Carried out at step = 730. Actorloss = 0.1475900113582611, Critic_loss = 0.09372076392173767\n",
      "Episode 10. Training Carried out at step = 740. Actorloss = 0.14653751254081726, Critic_loss = 0.05262858420610428\n",
      "Episode 10. Training Carried out at step = 750. Actorloss = 0.14645789563655853, Critic_loss = 0.04734909534454346\n",
      "Episode 10. Training Carried out at step = 760. Actorloss = 0.14627033472061157, Critic_loss = 0.04539027065038681\n",
      "Episode 10. Training Carried out at step = 770. Actorloss = 0.14551563560962677, Critic_loss = 0.0802004486322403\n",
      "Episode 10. Training Carried out at step = 780. Actorloss = 0.1451723575592041, Critic_loss = 0.09943005442619324\n",
      "Episode 10. Training Carried out at step = 790. Actorloss = 0.14503835141658783, Critic_loss = 0.06887167692184448\n",
      "Episode 10. Training Carried out at step = 800. Actorloss = 0.14464233815670013, Critic_loss = 0.01840837113559246\n",
      "Episode 10. Training Carried out at step = 810. Actorloss = 0.144262433052063, Critic_loss = 0.009653553366661072\n",
      "Episode 10. Training Carried out at step = 820. Actorloss = 0.14384351670742035, Critic_loss = 0.05151520296931267\n",
      "Episode 10. Training Carried out at step = 830. Actorloss = 0.14357970654964447, Critic_loss = 0.09542174637317657\n",
      "Episode 10. Training Carried out at step = 840. Actorloss = 0.1431346833705902, Critic_loss = 0.1269257813692093\n",
      "Episode 10. Training Carried out at step = 850. Actorloss = 0.14280767738819122, Critic_loss = 0.1638920158147812\n",
      "Episode 10. Training Carried out at step = 860. Actorloss = 0.14258699119091034, Critic_loss = 0.19855457544326782\n",
      "Episode 10. Training Carried out at step = 870. Actorloss = 0.14251039922237396, Critic_loss = 0.015835551545023918\n",
      "Episode 10. Training Carried out at step = 880. Actorloss = 0.14207005500793457, Critic_loss = 0.006657551042735577\n",
      "Episode 10. Training Carried out at step = 890. Actorloss = 0.14210925996303558, Critic_loss = 0.007931845262646675\n",
      "Episode 10. Training Carried out at step = 900. Actorloss = 0.14212878048419952, Critic_loss = 0.036532606929540634\n",
      "Episode 10. Training Carried out at step = 910. Actorloss = 0.14153648912906647, Critic_loss = 0.30508050322532654\n",
      "Episode 10. Training Carried out at step = 920. Actorloss = 0.14043018221855164, Critic_loss = 1.1444507837295532\n",
      "Episode 10. Training Carried out at step = 930. Actorloss = 0.1401643306016922, Critic_loss = 1.2826628684997559\n",
      "Episode 10. Training Carried out at step = 940. Actorloss = 0.13970141112804413, Critic_loss = 1.3022044897079468\n",
      "Episode 10. Training Carried out at step = 950. Actorloss = 0.13911204040050507, Critic_loss = 2.036515474319458\n",
      "Episode 10. Training Carried out at step = 960. Actorloss = 0.13788901269435883, Critic_loss = 2.5078699588775635\n",
      "Episode 10. Training Carried out at step = 970. Actorloss = 0.13775448501110077, Critic_loss = 1.1551765203475952\n",
      "Episode 10. Training Carried out at step = 980. Actorloss = 0.1379728466272354, Critic_loss = 0.005864426493644714\n",
      "Episode 10. Training Carried out at step = 990. Actorloss = 0.13715383410453796, Critic_loss = 0.015564627014100552\n",
      "Episode 10. Training Carried out at step = 1000. Actorloss = 0.13720403611660004, Critic_loss = 0.0650215670466423\n",
      "Episode 11. Training Carried out at step = 10. Actorloss = 0.13781321048736572, Critic_loss = 0.10064814984798431\n",
      "Episode 11. Training Carried out at step = 20. Actorloss = 0.14319996535778046, Critic_loss = 0.33167552947998047\n",
      "Episode 11. Training Carried out at step = 30. Actorloss = 0.13777856528759003, Critic_loss = 4.725487232208252\n",
      "Episode 11. Training Carried out at step = 40. Actorloss = 0.14358696341514587, Critic_loss = 2.3268444538116455\n",
      "Episode 11. Training Carried out at step = 50. Actorloss = 0.1447010189294815, Critic_loss = 1.2320475578308105\n",
      "Episode 11. Training Carried out at step = 60. Actorloss = 0.14517034590244293, Critic_loss = 0.5323494672775269\n",
      "Episode 11. Training Carried out at step = 70. Actorloss = 0.1388779580593109, Critic_loss = 3.542389392852783\n",
      "Episode 11. Training Carried out at step = 80. Actorloss = 0.14423532783985138, Critic_loss = 1.2441731691360474\n",
      "Episode 11. Training Carried out at step = 90. Actorloss = 0.14525605738162994, Critic_loss = 0.06782549619674683\n",
      "Episode 11. Training Carried out at step = 100. Actorloss = 0.14684806764125824, Critic_loss = 0.08500862866640091\n",
      "Episode 11. Training Carried out at step = 110. Actorloss = 0.14957675337791443, Critic_loss = 0.40719857811927795\n",
      "Episode 11. Training Carried out at step = 120. Actorloss = 0.1465868353843689, Critic_loss = 0.996647298336029\n",
      "Episode 11. Training Carried out at step = 130. Actorloss = 0.14631876349449158, Critic_loss = 0.5515402555465698\n",
      "Episode 11. Training Carried out at step = 140. Actorloss = 0.14753352105617523, Critic_loss = 0.5743905901908875\n",
      "Episode 11. Training Carried out at step = 150. Actorloss = 0.150637686252594, Critic_loss = 0.07821746915578842\n",
      "Episode 11. Training Carried out at step = 160. Actorloss = 0.15230101346969604, Critic_loss = 0.05688090622425079\n",
      "Episode 11. Training Carried out at step = 170. Actorloss = 0.1510685682296753, Critic_loss = 0.307515949010849\n",
      "Episode 11. Training Carried out at step = 180. Actorloss = 0.1519307941198349, Critic_loss = 0.23683872818946838\n",
      "Episode 11. Training Carried out at step = 190. Actorloss = 0.15121616423130035, Critic_loss = 0.5969025492668152\n",
      "Episode 11. Training Carried out at step = 200. Actorloss = 0.15289604663848877, Critic_loss = 0.6513248085975647\n",
      "Episode 11. Training Carried out at step = 210. Actorloss = 0.1532929390668869, Critic_loss = 0.534935712814331\n",
      "Episode 11. Training Carried out at step = 220. Actorloss = 0.1545649766921997, Critic_loss = 0.5991782546043396\n",
      "Episode 11. Training Carried out at step = 230. Actorloss = 0.15613901615142822, Critic_loss = 0.5929466485977173\n",
      "Episode 11. Training Carried out at step = 240. Actorloss = 0.15440145134925842, Critic_loss = 0.23495160043239594\n",
      "Episode 11. Training Carried out at step = 250. Actorloss = 0.153283029794693, Critic_loss = 0.047193191945552826\n",
      "Episode 11. Training Carried out at step = 260. Actorloss = 0.15209931135177612, Critic_loss = 0.007233045529574156\n",
      "Episode 11. Training Carried out at step = 270. Actorloss = 0.1509907990694046, Critic_loss = 0.046060796827077866\n",
      "Episode 11. Training Carried out at step = 280. Actorloss = 0.15008428692817688, Critic_loss = 0.10797008126974106\n",
      "Episode 11. Training Carried out at step = 290. Actorloss = 0.15033544600009918, Critic_loss = 0.1711403727531433\n",
      "Episode 11. Training Carried out at step = 300. Actorloss = 0.14028118550777435, Critic_loss = 1.022875428199768\n",
      "Episode 11. Training Carried out at step = 310. Actorloss = 0.11111389100551605, Critic_loss = 12.897029876708984\n",
      "Episode 11. Training Carried out at step = 320. Actorloss = 0.1484883427619934, Critic_loss = 0.8102123141288757\n",
      "Episode 11. Training Carried out at step = 330. Actorloss = 0.16747447848320007, Critic_loss = 2.6090314388275146\n",
      "Episode 11. Training Carried out at step = 340. Actorloss = 0.15372775495052338, Critic_loss = 0.4408626854419708\n",
      "Episode 11. Training Carried out at step = 350. Actorloss = 0.1366567760705948, Critic_loss = 1.6113828420639038\n",
      "Episode 11. Training Carried out at step = 360. Actorloss = 0.13273127377033234, Critic_loss = 2.5418965816497803\n",
      "Episode 11. Training Carried out at step = 370. Actorloss = 0.151621013879776, Critic_loss = 0.11495024710893631\n",
      "Episode 11. Training Carried out at step = 380. Actorloss = 0.15110406279563904, Critic_loss = 0.1714807003736496\n",
      "Episode 11. Training Carried out at step = 390. Actorloss = 0.15074588358402252, Critic_loss = 0.1940794289112091\n",
      "Episode 11. Training Carried out at step = 400. Actorloss = 0.14840637147426605, Critic_loss = 0.194696307182312\n",
      "Episode 11. Training Carried out at step = 410. Actorloss = 0.14693671464920044, Critic_loss = 0.22376637160778046\n",
      "Episode 11. Training Carried out at step = 420. Actorloss = 0.1466449499130249, Critic_loss = 0.2820267975330353\n",
      "Episode 11. Training Carried out at step = 430. Actorloss = 0.14621780812740326, Critic_loss = 0.2823411226272583\n",
      "Episode 11. Training Carried out at step = 440. Actorloss = 0.15093573927879333, Critic_loss = 0.1269187033176422\n",
      "Episode 11. Training Carried out at step = 450. Actorloss = 0.15178528428077698, Critic_loss = 0.00552050955593586\n",
      "Episode 11. Training Carried out at step = 460. Actorloss = 0.15169066190719604, Critic_loss = 0.009202348999679089\n",
      "Episode 11. Training Carried out at step = 470. Actorloss = 0.1521545946598053, Critic_loss = 0.09381710737943649\n",
      "Episode 11. Training Carried out at step = 480. Actorloss = 0.1476830393075943, Critic_loss = 0.010467635467648506\n",
      "Episode 11. Training Carried out at step = 490. Actorloss = 0.14586269855499268, Critic_loss = 0.01661846600472927\n",
      "Episode 11. Training Carried out at step = 500. Actorloss = 0.14156535267829895, Critic_loss = 0.04863649979233742\n",
      "Episode 11. Training Carried out at step = 510. Actorloss = 0.14108799397945404, Critic_loss = 0.15555721521377563\n",
      "Episode 11. Training Carried out at step = 520. Actorloss = 0.1406315714120865, Critic_loss = 0.1403494030237198\n",
      "Episode 11. Training Carried out at step = 530. Actorloss = 0.14299710094928741, Critic_loss = 0.12535560131072998\n",
      "Episode 11. Training Carried out at step = 540. Actorloss = 0.13588282465934753, Critic_loss = 0.38532423973083496\n",
      "Episode 11. Training Carried out at step = 550. Actorloss = 0.13449683785438538, Critic_loss = 0.06394869834184647\n",
      "Episode 11. Training Carried out at step = 560. Actorloss = 0.13413852453231812, Critic_loss = 0.1082603707909584\n",
      "Episode 11. Training Carried out at step = 570. Actorloss = 0.1331346184015274, Critic_loss = 0.03409477323293686\n",
      "Episode 11. Training Carried out at step = 580. Actorloss = 0.13604794442653656, Critic_loss = 0.03075888566672802\n",
      "Episode 11. Training Carried out at step = 590. Actorloss = 0.13698406517505646, Critic_loss = 0.10631980001926422\n",
      "Episode 11. Training Carried out at step = 600. Actorloss = 0.13837535679340363, Critic_loss = 0.17059354484081268\n",
      "Episode 11. Training Carried out at step = 610. Actorloss = 0.13864225149154663, Critic_loss = 0.225179523229599\n",
      "Episode 11. Training Carried out at step = 620. Actorloss = 0.13861596584320068, Critic_loss = 0.27565157413482666\n",
      "Episode 11. Training Carried out at step = 630. Actorloss = 0.13947471976280212, Critic_loss = 0.319004088640213\n",
      "Episode 11. Training Carried out at step = 640. Actorloss = 0.14076906442642212, Critic_loss = 0.3951268196105957\n",
      "Episode 11. Training Carried out at step = 650. Actorloss = 0.14281708002090454, Critic_loss = 0.5518831610679626\n",
      "Episode 11. Training Carried out at step = 660. Actorloss = 0.14662469923496246, Critic_loss = 0.9904124736785889\n",
      "Episode 11. Training Carried out at step = 670. Actorloss = 0.1596778780221939, Critic_loss = 2.4092071056365967\n",
      "Episode 11. Training Carried out at step = 680. Actorloss = 0.14303083717823029, Critic_loss = 0.6339060068130493\n",
      "Episode 11. Training Carried out at step = 690. Actorloss = 0.1410657912492752, Critic_loss = 0.04358286410570145\n",
      "Episode 11. Training Carried out at step = 700. Actorloss = 0.13892249763011932, Critic_loss = 0.027554234489798546\n",
      "Episode 11. Training Carried out at step = 710. Actorloss = 0.13734497129917145, Critic_loss = 0.017488854005932808\n",
      "Episode 11. Training Carried out at step = 720. Actorloss = 0.13633833825588226, Critic_loss = 0.006085697561502457\n",
      "Episode 11. Training Carried out at step = 730. Actorloss = 0.1339370608329773, Critic_loss = 0.008907616138458252\n",
      "Episode 11. Training Carried out at step = 740. Actorloss = 0.1333404779434204, Critic_loss = 0.0344068706035614\n",
      "Episode 11. Training Carried out at step = 750. Actorloss = 0.13537728786468506, Critic_loss = 0.07174495607614517\n",
      "Episode 11. Training Carried out at step = 760. Actorloss = 0.14196208119392395, Critic_loss = 0.05674164369702339\n",
      "Episode 11. Training Carried out at step = 770. Actorloss = 0.140706866979599, Critic_loss = 0.07992491126060486\n",
      "Episode 11. Training Carried out at step = 780. Actorloss = 0.14195850491523743, Critic_loss = 0.209441676735878\n",
      "Episode 11. Training Carried out at step = 790. Actorloss = 0.14329153299331665, Critic_loss = 0.8134389519691467\n",
      "Episode 11. Training Carried out at step = 800. Actorloss = 0.13143153488636017, Critic_loss = 0.07385799288749695\n",
      "Episode 11. Training Carried out at step = 810. Actorloss = 0.13501818478107452, Critic_loss = 0.06701436638832092\n",
      "Episode 11. Training Carried out at step = 820. Actorloss = 0.13039037585258484, Critic_loss = 0.04120514169335365\n",
      "Episode 11. Training Carried out at step = 830. Actorloss = 0.12760108709335327, Critic_loss = 0.16150948405265808\n",
      "Episode 11. Training Carried out at step = 840. Actorloss = 0.13044436275959015, Critic_loss = 0.11784393340349197\n",
      "Episode 11. Training Carried out at step = 850. Actorloss = 0.12592534720897675, Critic_loss = 0.35176557302474976\n",
      "Episode 11. Training Carried out at step = 860. Actorloss = 0.11249466240406036, Critic_loss = 1.6797012090682983\n",
      "Episode 11. Training Carried out at step = 870. Actorloss = 0.11628644913434982, Critic_loss = 0.5435693860054016\n",
      "Episode 11. Training Carried out at step = 880. Actorloss = 0.11156074702739716, Critic_loss = 0.6748311519622803\n",
      "Episode 11. Training Carried out at step = 890. Actorloss = 0.12118222564458847, Critic_loss = 0.2601546049118042\n",
      "Episode 11. Training Carried out at step = 900. Actorloss = 0.1232549250125885, Critic_loss = 0.033231209963560104\n",
      "Episode 11. Training Carried out at step = 910. Actorloss = 0.1246878057718277, Critic_loss = 0.004797131754457951\n",
      "Episode 11. Training Carried out at step = 920. Actorloss = 0.1257556825876236, Critic_loss = 0.02934943325817585\n",
      "Episode 11. Training Carried out at step = 930. Actorloss = 0.12640716135501862, Critic_loss = 0.08416395634412766\n",
      "Episode 11. Training Carried out at step = 940. Actorloss = 0.12720219790935516, Critic_loss = 0.16637428104877472\n",
      "Episode 11. Training Carried out at step = 950. Actorloss = 0.12885810434818268, Critic_loss = 0.30495139956474304\n",
      "Episode 11. Training Carried out at step = 960. Actorloss = 0.1317601352930069, Critic_loss = 0.594092607498169\n",
      "Episode 11. Training Carried out at step = 970. Actorloss = 0.13744571805000305, Critic_loss = 1.7143073081970215\n",
      "Episode 11. Training Carried out at step = 980. Actorloss = 0.1450984925031662, Critic_loss = 1.3728002309799194\n",
      "Episode 11. Training Carried out at step = 990. Actorloss = 0.12717632949352264, Critic_loss = 0.40840286016464233\n",
      "Episode 11. Training Carried out at step = 1000. Actorloss = 0.1316121518611908, Critic_loss = 0.2839849293231964\n",
      "Episode 12. Training Carried out at step = 10. Actorloss = 0.14230918884277344, Critic_loss = 0.005156225059181452\n",
      "Episode 12. Training Carried out at step = 20. Actorloss = 0.13144457340240479, Critic_loss = 0.10494450479745865\n",
      "Episode 12. Training Carried out at step = 30. Actorloss = 0.10847353935241699, Critic_loss = 2.9597153663635254\n",
      "Episode 12. Training Carried out at step = 40. Actorloss = 0.13931535184383392, Critic_loss = 0.34784841537475586\n",
      "Episode 12. Training Carried out at step = 50. Actorloss = 0.11647554486989975, Critic_loss = 1.5034326314926147\n",
      "Episode 12. Training Carried out at step = 60. Actorloss = 0.15461301803588867, Critic_loss = 0.8031096458435059\n",
      "Episode 12. Training Carried out at step = 70. Actorloss = 0.1552434265613556, Critic_loss = 0.7327173352241516\n",
      "Episode 12. Training Carried out at step = 80. Actorloss = 0.1443609893321991, Critic_loss = 0.4624652862548828\n",
      "Episode 12. Training Carried out at step = 90. Actorloss = 0.14689049124717712, Critic_loss = 0.038668543100357056\n",
      "Episode 12. Training Carried out at step = 100. Actorloss = 0.14680559933185577, Critic_loss = 0.040512483566999435\n",
      "Episode 12. Training Carried out at step = 110. Actorloss = 0.14646144211292267, Critic_loss = 0.05962195619940758\n",
      "Episode 12. Training Carried out at step = 120. Actorloss = 0.1460275799036026, Critic_loss = 0.290872186422348\n",
      "Episode 12. Training Carried out at step = 130. Actorloss = 0.1445416361093521, Critic_loss = 0.24417170882225037\n",
      "Episode 12. Training Carried out at step = 140. Actorloss = 0.1474110335111618, Critic_loss = 0.11668675392866135\n",
      "Episode 12. Training Carried out at step = 150. Actorloss = 0.15079021453857422, Critic_loss = 0.10832397639751434\n",
      "Episode 12. Training Carried out at step = 160. Actorloss = 0.15082301199436188, Critic_loss = 0.2023753821849823\n",
      "Episode 12. Training Carried out at step = 170. Actorloss = 0.15105272829532623, Critic_loss = 0.24682307243347168\n",
      "Episode 12. Training Carried out at step = 180. Actorloss = 0.14788676798343658, Critic_loss = 0.4926639497280121\n",
      "Episode 12. Training Carried out at step = 190. Actorloss = 0.14199213683605194, Critic_loss = 1.3715027570724487\n",
      "Episode 12. Training Carried out at step = 200. Actorloss = 0.14796467125415802, Critic_loss = 0.40593379735946655\n",
      "Episode 12. Training Carried out at step = 210. Actorloss = 0.14765314757823944, Critic_loss = 0.36342793703079224\n",
      "Episode 12. Training Carried out at step = 220. Actorloss = 0.150427907705307, Critic_loss = 0.16109269857406616\n",
      "Episode 12. Training Carried out at step = 230. Actorloss = 0.1503525823354721, Critic_loss = 0.03768812492489815\n",
      "Episode 12. Training Carried out at step = 240. Actorloss = 0.14822359383106232, Critic_loss = 0.011235525831580162\n",
      "Episode 12. Training Carried out at step = 250. Actorloss = 0.14585107564926147, Critic_loss = 0.004121710080653429\n",
      "Episode 12. Training Carried out at step = 260. Actorloss = 0.14619697630405426, Critic_loss = 0.003909960389137268\n",
      "Episode 12. Training Carried out at step = 270. Actorloss = 0.1475616991519928, Critic_loss = 0.0034246251452714205\n",
      "Episode 12. Training Carried out at step = 280. Actorloss = 0.14654995501041412, Critic_loss = 0.005006086081266403\n",
      "Episode 12. Training Carried out at step = 290. Actorloss = 0.14662966132164001, Critic_loss = 0.016789676621556282\n",
      "Episode 12. Training Carried out at step = 300. Actorloss = 0.14563417434692383, Critic_loss = 0.06963364034891129\n",
      "Episode 12. Training Carried out at step = 310. Actorloss = 0.14278721809387207, Critic_loss = 0.06634683161973953\n",
      "Episode 12. Training Carried out at step = 320. Actorloss = 0.14043407142162323, Critic_loss = 0.14222456514835358\n",
      "Episode 12. Training Carried out at step = 330. Actorloss = 0.13829049468040466, Critic_loss = 0.21914486587047577\n",
      "Episode 12. Training Carried out at step = 340. Actorloss = 0.13957445323467255, Critic_loss = 0.35491088032722473\n",
      "Episode 12. Training Carried out at step = 350. Actorloss = 0.15090811252593994, Critic_loss = 1.4037350416183472\n",
      "Episode 12. Training Carried out at step = 360. Actorloss = 0.14472952485084534, Critic_loss = 0.9456722140312195\n",
      "Episode 12. Training Carried out at step = 370. Actorloss = 0.14209581911563873, Critic_loss = 0.43175023794174194\n",
      "Episode 12. Training Carried out at step = 380. Actorloss = 0.14304086565971375, Critic_loss = 0.035161420702934265\n",
      "Episode 12. Training Carried out at step = 390. Actorloss = 0.14016906917095184, Critic_loss = 0.029702527448534966\n",
      "Episode 12. Training Carried out at step = 400. Actorloss = 0.13698238134384155, Critic_loss = 0.021218322217464447\n",
      "Episode 12. Training Carried out at step = 410. Actorloss = 0.13557785749435425, Critic_loss = 0.009568492881953716\n",
      "Episode 12. Training Carried out at step = 420. Actorloss = 0.13559873402118683, Critic_loss = 0.005906297825276852\n",
      "Episode 12. Training Carried out at step = 430. Actorloss = 0.13514548540115356, Critic_loss = 0.019535820931196213\n",
      "Episode 12. Training Carried out at step = 440. Actorloss = 0.1346587985754013, Critic_loss = 0.0285648126155138\n",
      "Episode 12. Training Carried out at step = 450. Actorloss = 0.13398683071136475, Critic_loss = 0.029175326228141785\n",
      "Episode 12. Training Carried out at step = 460. Actorloss = 0.13344454765319824, Critic_loss = 0.019532475620508194\n",
      "Episode 12. Training Carried out at step = 470. Actorloss = 0.13308991491794586, Critic_loss = 0.011328681372106075\n",
      "Episode 12. Training Carried out at step = 480. Actorloss = 0.13358701765537262, Critic_loss = 0.06021679565310478\n",
      "Episode 12. Training Carried out at step = 490. Actorloss = 0.13574185967445374, Critic_loss = 0.04654490202665329\n",
      "Episode 12. Training Carried out at step = 500. Actorloss = 0.13737867772579193, Critic_loss = 0.021831734105944633\n",
      "Episode 12. Training Carried out at step = 510. Actorloss = 0.13997112214565277, Critic_loss = 0.010016199201345444\n",
      "Episode 12. Training Carried out at step = 520. Actorloss = 0.13368119299411774, Critic_loss = 0.11637134104967117\n",
      "Episode 12. Training Carried out at step = 530. Actorloss = 0.12695527076721191, Critic_loss = 0.051604971289634705\n",
      "Episode 12. Training Carried out at step = 540. Actorloss = 0.12940789759159088, Critic_loss = 0.010610904544591904\n",
      "Episode 12. Training Carried out at step = 550. Actorloss = 0.13180692493915558, Critic_loss = 0.06303268671035767\n",
      "Episode 12. Training Carried out at step = 560. Actorloss = 0.13880877196788788, Critic_loss = 0.36900943517684937\n",
      "Episode 12. Training Carried out at step = 570. Actorloss = 0.15496626496315002, Critic_loss = 1.1946260929107666\n",
      "Episode 12. Training Carried out at step = 580. Actorloss = 0.16810597479343414, Critic_loss = 2.419572591781616\n",
      "Episode 12. Training Carried out at step = 590. Actorloss = 0.14192184805870056, Critic_loss = 0.24564991891384125\n",
      "Episode 12. Training Carried out at step = 600. Actorloss = 0.14085005223751068, Critic_loss = 0.22562754154205322\n",
      "Episode 12. Training Carried out at step = 610. Actorloss = 0.1388154923915863, Critic_loss = 0.15055117011070251\n",
      "Episode 12. Training Carried out at step = 620. Actorloss = 0.13610371947288513, Critic_loss = 0.055782876908779144\n",
      "Episode 12. Training Carried out at step = 630. Actorloss = 0.13436877727508545, Critic_loss = 0.007485206704586744\n",
      "Episode 12. Training Carried out at step = 640. Actorloss = 0.13195183873176575, Critic_loss = 0.028318040072917938\n",
      "Episode 12. Training Carried out at step = 650. Actorloss = 0.12869086861610413, Critic_loss = 0.1722871959209442\n",
      "Episode 12. Training Carried out at step = 660. Actorloss = 0.12750057876110077, Critic_loss = 0.43395915627479553\n",
      "Episode 12. Training Carried out at step = 670. Actorloss = 0.12465880811214447, Critic_loss = 0.9748762845993042\n",
      "Episode 12. Training Carried out at step = 680. Actorloss = 0.1185537800192833, Critic_loss = 1.763218641281128\n",
      "Episode 12. Training Carried out at step = 690. Actorloss = 0.10450556874275208, Critic_loss = 3.019869327545166\n",
      "Episode 12. Training Carried out at step = 700. Actorloss = 0.10154552012681961, Critic_loss = 3.1338326930999756\n",
      "Episode 12. Training Carried out at step = 710. Actorloss = 0.12496548146009445, Critic_loss = 0.013883711770176888\n",
      "Episode 12. Training Carried out at step = 720. Actorloss = 0.12242216616868973, Critic_loss = 0.004334497265517712\n",
      "Episode 12. Training Carried out at step = 730. Actorloss = 0.11809933930635452, Critic_loss = 0.003792861010879278\n",
      "Episode 12. Training Carried out at step = 740. Actorloss = 0.11797617375850677, Critic_loss = 0.0036346460692584515\n",
      "Episode 12. Training Carried out at step = 750. Actorloss = 0.11652351915836334, Critic_loss = 0.005440611392259598\n",
      "Episode 12. Training Carried out at step = 760. Actorloss = 0.11588074266910553, Critic_loss = 0.006436829920858145\n",
      "Episode 12. Training Carried out at step = 770. Actorloss = 0.11491208523511887, Critic_loss = 0.007571062073111534\n",
      "Episode 12. Training Carried out at step = 780. Actorloss = 0.11377385258674622, Critic_loss = 0.008361534215509892\n",
      "Episode 12. Training Carried out at step = 790. Actorloss = 0.1126629039645195, Critic_loss = 0.008583752438426018\n",
      "Episode 12. Training Carried out at step = 800. Actorloss = 0.11162101477384567, Critic_loss = 0.00819338858127594\n",
      "Episode 12. Training Carried out at step = 810. Actorloss = 0.11075389385223389, Critic_loss = 0.009816081263124943\n",
      "Episode 12. Training Carried out at step = 820. Actorloss = 0.09915994852781296, Critic_loss = 0.009070854634046555\n",
      "Episode 12. Training Carried out at step = 830. Actorloss = 0.09440774470567703, Critic_loss = 0.0032959652598947287\n",
      "Episode 12. Training Carried out at step = 840. Actorloss = 0.09471560269594193, Critic_loss = 0.011317554861307144\n",
      "Episode 12. Training Carried out at step = 850. Actorloss = 0.09461788088083267, Critic_loss = 0.024296993389725685\n",
      "Episode 12. Training Carried out at step = 860. Actorloss = 0.09266981482505798, Critic_loss = 0.03541657701134682\n",
      "Episode 12. Training Carried out at step = 870. Actorloss = 0.09298314154148102, Critic_loss = 0.05565253645181656\n",
      "Episode 12. Training Carried out at step = 880. Actorloss = 0.09285353869199753, Critic_loss = 0.08831670880317688\n",
      "Episode 12. Training Carried out at step = 890. Actorloss = 0.10305241495370865, Critic_loss = 0.08664268255233765\n",
      "Episode 12. Training Carried out at step = 900. Actorloss = 0.10112763196229935, Critic_loss = 0.2074519842863083\n",
      "Episode 12. Training Carried out at step = 910. Actorloss = 0.09910693019628525, Critic_loss = 0.332600474357605\n",
      "Episode 12. Training Carried out at step = 920. Actorloss = 0.09677080065011978, Critic_loss = 0.4680689573287964\n",
      "Episode 12. Training Carried out at step = 930. Actorloss = 0.09500206261873245, Critic_loss = 0.5468859672546387\n",
      "Episode 12. Training Carried out at step = 940. Actorloss = 0.09350141137838364, Critic_loss = 0.5782699584960938\n",
      "Episode 12. Training Carried out at step = 950. Actorloss = 0.09148068726062775, Critic_loss = 0.5764897465705872\n",
      "Episode 12. Training Carried out at step = 960. Actorloss = 0.09112100303173065, Critic_loss = 0.6942415833473206\n",
      "Episode 12. Training Carried out at step = 970. Actorloss = 0.08828910440206528, Critic_loss = 1.2419883012771606\n",
      "Episode 12. Training Carried out at step = 980. Actorloss = 0.08567959815263748, Critic_loss = 1.4217334985733032\n",
      "Episode 12. Training Carried out at step = 990. Actorloss = 0.0838387981057167, Critic_loss = 0.6376272439956665\n",
      "Episode 12. Training Carried out at step = 1000. Actorloss = 0.08539760112762451, Critic_loss = 0.12387515604496002\n",
      "Episode 13. Training Carried out at step = 10. Actorloss = 0.1251830905675888, Critic_loss = 0.3982187509536743\n",
      "Episode 13. Training Carried out at step = 20. Actorloss = 0.10156320035457611, Critic_loss = 0.4636473059654236\n",
      "Episode 13. Training Carried out at step = 30. Actorloss = 0.0854809358716011, Critic_loss = 0.671321451663971\n",
      "Episode 13. Training Carried out at step = 40. Actorloss = 0.1186806783080101, Critic_loss = 0.333222359418869\n",
      "Episode 13. Training Carried out at step = 50. Actorloss = 0.0832110121846199, Critic_loss = 0.2745499610900879\n",
      "Episode 13. Training Carried out at step = 60. Actorloss = 0.125451922416687, Critic_loss = 1.3768199682235718\n",
      "Episode 13. Training Carried out at step = 70. Actorloss = 0.1733626425266266, Critic_loss = 4.009021759033203\n",
      "Episode 13. Training Carried out at step = 80. Actorloss = 0.1571105718612671, Critic_loss = 1.1274546384811401\n",
      "Episode 13. Training Carried out at step = 90. Actorloss = 0.16454869508743286, Critic_loss = 1.587583065032959\n",
      "Episode 13. Training Carried out at step = 100. Actorloss = 0.2031223326921463, Critic_loss = 4.100342750549316\n",
      "Episode 13. Training Carried out at step = 110. Actorloss = 0.1303716003894806, Critic_loss = 1.334351658821106\n",
      "Episode 13. Training Carried out at step = 120. Actorloss = 0.13631345331668854, Critic_loss = 0.40430235862731934\n",
      "Episode 13. Training Carried out at step = 130. Actorloss = 0.13133488595485687, Critic_loss = 0.36599427461624146\n",
      "Episode 13. Training Carried out at step = 140. Actorloss = 0.13823844492435455, Critic_loss = 0.2873082756996155\n",
      "Episode 13. Training Carried out at step = 150. Actorloss = 0.14620834589004517, Critic_loss = 0.21204505860805511\n",
      "Episode 13. Training Carried out at step = 160. Actorloss = 0.1445252001285553, Critic_loss = 0.20439988374710083\n",
      "Episode 13. Training Carried out at step = 170. Actorloss = 0.14592380821704865, Critic_loss = 0.17834258079528809\n",
      "Episode 13. Training Carried out at step = 180. Actorloss = 0.14563053846359253, Critic_loss = 0.18583498895168304\n",
      "Episode 13. Training Carried out at step = 190. Actorloss = 0.13668473064899445, Critic_loss = 0.21896158158779144\n",
      "Episode 13. Training Carried out at step = 200. Actorloss = 0.12698407471179962, Critic_loss = 0.1273328959941864\n",
      "Episode 13. Training Carried out at step = 210. Actorloss = 0.12436860054731369, Critic_loss = 0.08483587205410004\n",
      "Episode 13. Training Carried out at step = 220. Actorloss = 0.11679702997207642, Critic_loss = 0.08932673186063766\n",
      "Episode 13. Training Carried out at step = 230. Actorloss = 0.11915407329797745, Critic_loss = 0.06684525310993195\n",
      "Episode 13. Training Carried out at step = 240. Actorloss = 0.1183396503329277, Critic_loss = 0.0608297623693943\n",
      "Episode 13. Training Carried out at step = 250. Actorloss = 0.12115687131881714, Critic_loss = 0.07474619895219803\n",
      "Episode 13. Training Carried out at step = 260. Actorloss = 0.13137158751487732, Critic_loss = 0.15907777845859528\n",
      "Episode 13. Training Carried out at step = 270. Actorloss = 0.12688466906547546, Critic_loss = 0.008157678879797459\n",
      "Episode 13. Training Carried out at step = 280. Actorloss = 0.1242418959736824, Critic_loss = 0.040270835161209106\n",
      "Episode 13. Training Carried out at step = 290. Actorloss = 0.12053146213293076, Critic_loss = 0.16099679470062256\n",
      "Episode 13. Training Carried out at step = 300. Actorloss = 0.11862825602293015, Critic_loss = 0.3396744132041931\n",
      "Episode 13. Training Carried out at step = 310. Actorloss = 0.11731880158185959, Critic_loss = 0.4340928792953491\n",
      "Episode 13. Training Carried out at step = 320. Actorloss = 0.11470730602741241, Critic_loss = 0.4212452173233032\n",
      "Episode 13. Training Carried out at step = 330. Actorloss = 0.11278786510229111, Critic_loss = 0.6996830105781555\n",
      "Episode 13. Training Carried out at step = 340. Actorloss = 0.10831400752067566, Critic_loss = 1.4307572841644287\n",
      "Episode 13. Training Carried out at step = 350. Actorloss = 0.10419264435768127, Critic_loss = 1.5642657279968262\n",
      "Episode 13. Training Carried out at step = 360. Actorloss = 0.12100889533758163, Critic_loss = 0.38144978880882263\n",
      "Episode 13. Training Carried out at step = 370. Actorloss = 0.1082078143954277, Critic_loss = 0.14133326709270477\n",
      "Episode 13. Training Carried out at step = 380. Actorloss = 0.10676330327987671, Critic_loss = 0.10789316892623901\n",
      "Episode 13. Training Carried out at step = 390. Actorloss = 0.1008603423833847, Critic_loss = 0.17754516005516052\n",
      "Episode 13. Training Carried out at step = 400. Actorloss = 0.09109645336866379, Critic_loss = 0.6653779745101929\n",
      "Episode 13. Training Carried out at step = 410. Actorloss = 0.08599995076656342, Critic_loss = 1.510839819908142\n",
      "Episode 13. Training Carried out at step = 420. Actorloss = 0.08822806179523468, Critic_loss = 1.1479613780975342\n",
      "Episode 13. Training Carried out at step = 430. Actorloss = 0.08938073366880417, Critic_loss = 0.5717471837997437\n",
      "Episode 13. Training Carried out at step = 440. Actorloss = 0.11503675580024719, Critic_loss = 0.3311488628387451\n",
      "Episode 13. Training Carried out at step = 450. Actorloss = 0.10132284462451935, Critic_loss = 0.04633411765098572\n",
      "Episode 13. Training Carried out at step = 460. Actorloss = 0.10173256695270538, Critic_loss = 0.031278111040592194\n",
      "Episode 13. Training Carried out at step = 470. Actorloss = 0.09580283612012863, Critic_loss = 0.0829721987247467\n",
      "Episode 13. Training Carried out at step = 480. Actorloss = 0.09582571685314178, Critic_loss = 0.05526309832930565\n",
      "Episode 13. Training Carried out at step = 490. Actorloss = 0.09617242217063904, Critic_loss = 0.015405476093292236\n",
      "Episode 13. Training Carried out at step = 500. Actorloss = 0.09095855802297592, Critic_loss = 0.11860626190900803\n",
      "Episode 13. Training Carried out at step = 510. Actorloss = 0.09146057069301605, Critic_loss = 0.364092081785202\n",
      "Episode 13. Training Carried out at step = 520. Actorloss = 0.09281320124864578, Critic_loss = 0.0033641392365098\n",
      "Episode 13. Training Carried out at step = 530. Actorloss = 0.09663490206003189, Critic_loss = 0.050213951617479324\n",
      "Episode 13. Training Carried out at step = 540. Actorloss = 0.09997529536485672, Critic_loss = 0.1308949589729309\n",
      "Episode 13. Training Carried out at step = 550. Actorloss = 0.09722727537155151, Critic_loss = 0.141593337059021\n",
      "Episode 13. Training Carried out at step = 560. Actorloss = 0.09466040134429932, Critic_loss = 0.16432227194309235\n",
      "Episode 13. Training Carried out at step = 570. Actorloss = 0.09445524960756302, Critic_loss = 0.1428278535604477\n",
      "Episode 13. Training Carried out at step = 580. Actorloss = 0.09424912184476852, Critic_loss = 0.13027004897594452\n",
      "Episode 13. Training Carried out at step = 590. Actorloss = 0.0935552641749382, Critic_loss = 0.122334785759449\n",
      "Episode 13. Training Carried out at step = 600. Actorloss = 0.09317043423652649, Critic_loss = 0.10943332314491272\n",
      "Episode 13. Training Carried out at step = 610. Actorloss = 0.09314291179180145, Critic_loss = 0.14020904898643494\n",
      "Episode 13. Training Carried out at step = 620. Actorloss = 0.09299316257238388, Critic_loss = 0.19892409443855286\n",
      "Episode 13. Training Carried out at step = 630. Actorloss = 0.0905289351940155, Critic_loss = 0.2932560443878174\n",
      "Episode 13. Training Carried out at step = 640. Actorloss = 0.09006956964731216, Critic_loss = 0.3856770396232605\n",
      "Episode 13. Training Carried out at step = 650. Actorloss = 0.09312725812196732, Critic_loss = 0.8903738260269165\n",
      "Episode 13. Training Carried out at step = 660. Actorloss = 0.09623215347528458, Critic_loss = 1.1626126766204834\n",
      "Episode 13. Training Carried out at step = 670. Actorloss = 0.09031236171722412, Critic_loss = 0.4269341230392456\n",
      "Episode 13. Training Carried out at step = 680. Actorloss = 0.09144467860460281, Critic_loss = 0.46104100346565247\n",
      "Episode 13. Training Carried out at step = 690. Actorloss = 0.09204103797674179, Critic_loss = 0.13049395382404327\n",
      "Episode 13. Training Carried out at step = 700. Actorloss = 0.09287206828594208, Critic_loss = 0.021825307980179787\n",
      "Episode 13. Training Carried out at step = 710. Actorloss = 0.09217838943004608, Critic_loss = 0.009861327707767487\n",
      "Episode 13. Training Carried out at step = 720. Actorloss = 0.09086675196886063, Critic_loss = 0.004252329934388399\n",
      "Episode 13. Training Carried out at step = 730. Actorloss = 0.09069753438234329, Critic_loss = 0.004241517279297113\n",
      "Episode 13. Training Carried out at step = 740. Actorloss = 0.09045401215553284, Critic_loss = 0.02661558985710144\n",
      "Episode 13. Training Carried out at step = 750. Actorloss = 0.09034805744886398, Critic_loss = 0.04177560657262802\n",
      "Episode 13. Training Carried out at step = 760. Actorloss = 0.08968102931976318, Critic_loss = 0.03242375701665878\n",
      "Episode 13. Training Carried out at step = 770. Actorloss = 0.08797555416822433, Critic_loss = 0.011401508934795856\n",
      "Episode 13. Training Carried out at step = 780. Actorloss = 0.08799565583467484, Critic_loss = 0.0054325442761182785\n",
      "Episode 13. Training Carried out at step = 790. Actorloss = 0.08806101977825165, Critic_loss = 0.023540418595075607\n",
      "Episode 13. Training Carried out at step = 800. Actorloss = 0.08740831166505814, Critic_loss = 0.08057346940040588\n",
      "Episode 13. Training Carried out at step = 810. Actorloss = 0.08586254715919495, Critic_loss = 0.5798231363296509\n",
      "Episode 13. Training Carried out at step = 820. Actorloss = 0.08721502870321274, Critic_loss = 0.08488267660140991\n",
      "Episode 13. Training Carried out at step = 830. Actorloss = 0.08692994713783264, Critic_loss = 0.09162269532680511\n",
      "Episode 13. Training Carried out at step = 840. Actorloss = 0.08669622987508774, Critic_loss = 0.08020932972431183\n",
      "Episode 13. Training Carried out at step = 850. Actorloss = 0.0891861692070961, Critic_loss = 0.030703144147992134\n",
      "Episode 13. Training Carried out at step = 860. Actorloss = 0.08915204554796219, Critic_loss = 0.025017322972416878\n",
      "Episode 13. Training Carried out at step = 870. Actorloss = 0.08753913640975952, Critic_loss = 0.039596110582351685\n",
      "Episode 13. Training Carried out at step = 880. Actorloss = 0.0871768444776535, Critic_loss = 0.03744475543498993\n",
      "Episode 13. Training Carried out at step = 890. Actorloss = 0.08699925243854523, Critic_loss = 0.036106497049331665\n",
      "Episode 13. Training Carried out at step = 900. Actorloss = 0.08699271827936172, Critic_loss = 0.03568677976727486\n",
      "Episode 13. Training Carried out at step = 910. Actorloss = 0.08671941608190536, Critic_loss = 0.07532399147748947\n",
      "Episode 13. Training Carried out at step = 920. Actorloss = 0.08660424500703812, Critic_loss = 0.17814789712429047\n",
      "Episode 13. Training Carried out at step = 930. Actorloss = 0.08619234710931778, Critic_loss = 0.1628531962633133\n",
      "Episode 13. Training Carried out at step = 940. Actorloss = 0.08576499670743942, Critic_loss = 0.14466990530490875\n",
      "Episode 13. Training Carried out at step = 950. Actorloss = 0.08580167591571808, Critic_loss = 0.23233675956726074\n",
      "Episode 13. Training Carried out at step = 960. Actorloss = 0.0873412936925888, Critic_loss = 0.8103451728820801\n",
      "Episode 13. Training Carried out at step = 970. Actorloss = 0.08584710955619812, Critic_loss = 0.362358957529068\n",
      "Episode 13. Training Carried out at step = 980. Actorloss = 0.08585435897111893, Critic_loss = 0.1756802499294281\n",
      "Episode 13. Training Carried out at step = 990. Actorloss = 0.08577752113342285, Critic_loss = 0.21364864706993103\n",
      "Episode 13. Training Carried out at step = 1000. Actorloss = 0.08605417609214783, Critic_loss = 0.1016162857413292\n",
      "Episode 14. Training Carried out at step = 10. Actorloss = 0.10905595868825912, Critic_loss = 0.004370331764221191\n",
      "Episode 14. Training Carried out at step = 20. Actorloss = 0.09618718177080154, Critic_loss = 0.09694014489650726\n",
      "Episode 14. Training Carried out at step = 30. Actorloss = 0.09559394419193268, Critic_loss = 2.341810941696167\n",
      "Episode 14. Training Carried out at step = 40. Actorloss = 0.10962388664484024, Critic_loss = 0.08933383971452713\n",
      "Episode 14. Training Carried out at step = 50. Actorloss = 0.09833375364542007, Critic_loss = 0.8449749946594238\n",
      "Episode 14. Training Carried out at step = 60. Actorloss = 0.13880814611911774, Critic_loss = 0.9239218831062317\n",
      "Episode 14. Training Carried out at step = 70. Actorloss = 0.11865704506635666, Critic_loss = 0.4383348822593689\n",
      "Episode 14. Training Carried out at step = 80. Actorloss = 0.11637409031391144, Critic_loss = 0.16197150945663452\n",
      "Episode 14. Training Carried out at step = 90. Actorloss = 0.1176033616065979, Critic_loss = 0.04345247149467468\n",
      "Episode 14. Training Carried out at step = 100. Actorloss = 0.11045518517494202, Critic_loss = 0.04336407035589218\n",
      "Episode 14. Training Carried out at step = 110. Actorloss = 0.1084592416882515, Critic_loss = 0.05563738942146301\n",
      "Episode 14. Training Carried out at step = 120. Actorloss = 0.10780943185091019, Critic_loss = 0.2788154184818268\n",
      "Episode 14. Training Carried out at step = 130. Actorloss = 0.10520554333925247, Critic_loss = 0.30914756655693054\n",
      "Episode 14. Training Carried out at step = 140. Actorloss = 0.10920809209346771, Critic_loss = 0.1366821676492691\n",
      "Episode 14. Training Carried out at step = 150. Actorloss = 0.11574851721525192, Critic_loss = 0.13770997524261475\n",
      "Episode 14. Training Carried out at step = 160. Actorloss = 0.11471053957939148, Critic_loss = 0.2255312204360962\n",
      "Episode 14. Training Carried out at step = 170. Actorloss = 0.11441822350025177, Critic_loss = 0.33822810649871826\n",
      "Episode 14. Training Carried out at step = 180. Actorloss = 0.10183316469192505, Critic_loss = 1.1090834140777588\n",
      "Episode 14. Training Carried out at step = 190. Actorloss = 0.11135226488113403, Critic_loss = 0.6784482002258301\n",
      "Episode 14. Training Carried out at step = 200. Actorloss = 0.10864458978176117, Critic_loss = 0.5005188584327698\n",
      "Episode 14. Training Carried out at step = 210. Actorloss = 0.10823021084070206, Critic_loss = 0.47241830825805664\n",
      "Episode 14. Training Carried out at step = 220. Actorloss = 0.11980962008237839, Critic_loss = 0.2897067070007324\n",
      "Episode 14. Training Carried out at step = 230. Actorloss = 0.12115221470594406, Critic_loss = 0.09720644354820251\n",
      "Episode 14. Training Carried out at step = 240. Actorloss = 0.11865489929914474, Critic_loss = 0.05566907301545143\n",
      "Episode 14. Training Carried out at step = 250. Actorloss = 0.11205856502056122, Critic_loss = 0.04016575217247009\n",
      "Episode 14. Training Carried out at step = 260. Actorloss = 0.1116388812661171, Critic_loss = 0.033932603895664215\n",
      "Episode 14. Training Carried out at step = 270. Actorloss = 0.1111990213394165, Critic_loss = 0.025882458314299583\n",
      "Episode 14. Training Carried out at step = 280. Actorloss = 0.11022411286830902, Critic_loss = 0.007569588255137205\n",
      "Episode 14. Training Carried out at step = 290. Actorloss = 0.10798463225364685, Critic_loss = 0.015468021854758263\n",
      "Episode 14. Training Carried out at step = 300. Actorloss = 0.10654585063457489, Critic_loss = 0.01991795003414154\n",
      "Episode 14. Training Carried out at step = 310. Actorloss = 0.10251884162425995, Critic_loss = 0.059864241629838943\n",
      "Episode 14. Training Carried out at step = 320. Actorloss = 0.10174057632684708, Critic_loss = 0.14825904369354248\n",
      "Episode 14. Training Carried out at step = 330. Actorloss = 0.10460773855447769, Critic_loss = 0.3497356176376343\n",
      "Episode 14. Training Carried out at step = 340. Actorloss = 0.11559710651636124, Critic_loss = 0.8786289095878601\n",
      "Episode 14. Training Carried out at step = 350. Actorloss = 0.10235200077295303, Critic_loss = 0.2459222823381424\n",
      "Episode 14. Training Carried out at step = 360. Actorloss = 0.10179842263460159, Critic_loss = 0.26392802596092224\n",
      "Episode 14. Training Carried out at step = 370. Actorloss = 0.10377170145511627, Critic_loss = 0.173815056681633\n",
      "Episode 14. Training Carried out at step = 380. Actorloss = 0.10734965652227402, Critic_loss = 0.03385010361671448\n",
      "Episode 14. Training Carried out at step = 390. Actorloss = 0.10378403216600418, Critic_loss = 0.019159706309437752\n",
      "Episode 14. Training Carried out at step = 400. Actorloss = 0.10168275982141495, Critic_loss = 0.005929380655288696\n",
      "Episode 14. Training Carried out at step = 410. Actorloss = 0.10269089788198471, Critic_loss = 0.02846331149339676\n",
      "Episode 14. Training Carried out at step = 420. Actorloss = 0.10240192711353302, Critic_loss = 0.0511007085442543\n",
      "Episode 14. Training Carried out at step = 430. Actorloss = 0.1017378568649292, Critic_loss = 0.05907946079969406\n",
      "Episode 14. Training Carried out at step = 440. Actorloss = 0.09975754469633102, Critic_loss = 0.042531564831733704\n",
      "Episode 14. Training Carried out at step = 450. Actorloss = 0.09908092021942139, Critic_loss = 0.02998535893857479\n",
      "Episode 14. Training Carried out at step = 460. Actorloss = 0.09874227643013, Critic_loss = 0.012484477832913399\n",
      "Episode 14. Training Carried out at step = 470. Actorloss = 0.09879907220602036, Critic_loss = 0.018824739381670952\n",
      "Episode 14. Training Carried out at step = 480. Actorloss = 0.09876326471567154, Critic_loss = 0.050128936767578125\n",
      "Episode 14. Training Carried out at step = 490. Actorloss = 0.09906868636608124, Critic_loss = 0.07917772978544235\n",
      "Episode 14. Training Carried out at step = 500. Actorloss = 0.09624918550252914, Critic_loss = 0.25992289185523987\n",
      "Episode 14. Training Carried out at step = 510. Actorloss = 0.09567756205797195, Critic_loss = 0.961877703666687\n",
      "Episode 14. Training Carried out at step = 520. Actorloss = 0.09626875072717667, Critic_loss = 0.23697394132614136\n",
      "Episode 14. Training Carried out at step = 530. Actorloss = 0.09871888905763626, Critic_loss = 0.1899271160364151\n",
      "Episode 14. Training Carried out at step = 540. Actorloss = 0.09929016977548599, Critic_loss = 0.05125872418284416\n",
      "Episode 14. Training Carried out at step = 550. Actorloss = 0.09989774972200394, Critic_loss = 0.017426718026399612\n",
      "Episode 14. Training Carried out at step = 560. Actorloss = 0.09754327684640884, Critic_loss = 0.0032883791718631983\n",
      "Episode 14. Training Carried out at step = 570. Actorloss = 0.09772669523954391, Critic_loss = 0.0029010060243308544\n",
      "Episode 14. Training Carried out at step = 580. Actorloss = 0.09725048393011093, Critic_loss = 0.003921847324818373\n",
      "Episode 14. Training Carried out at step = 590. Actorloss = 0.09703708440065384, Critic_loss = 0.005233229137957096\n",
      "Episode 14. Training Carried out at step = 600. Actorloss = 0.09658115357160568, Critic_loss = 0.01614663377404213\n",
      "Episode 14. Training Carried out at step = 610. Actorloss = 0.09609425812959671, Critic_loss = 0.05822800099849701\n",
      "Episode 14. Training Carried out at step = 620. Actorloss = 0.09590786695480347, Critic_loss = 0.0293960589915514\n",
      "Episode 14. Training Carried out at step = 630. Actorloss = 0.09581228345632553, Critic_loss = 0.09099577367305756\n",
      "Episode 14. Training Carried out at step = 640. Actorloss = 0.09440373629331589, Critic_loss = 0.20971174538135529\n",
      "Episode 14. Training Carried out at step = 650. Actorloss = 0.09442413598299026, Critic_loss = 0.2927361726760864\n",
      "Episode 14. Training Carried out at step = 660. Actorloss = 0.09790351986885071, Critic_loss = 0.9730088710784912\n",
      "Episode 14. Training Carried out at step = 670. Actorloss = 0.09835118800401688, Critic_loss = 0.9827196002006531\n",
      "Episode 14. Training Carried out at step = 680. Actorloss = 0.09470725804567337, Critic_loss = 0.3227388858795166\n",
      "Episode 14. Training Carried out at step = 690. Actorloss = 0.09636089205741882, Critic_loss = 0.0595264695584774\n",
      "Episode 14. Training Carried out at step = 700. Actorloss = 0.09609902650117874, Critic_loss = 0.045080557465553284\n",
      "Episode 14. Training Carried out at step = 710. Actorloss = 0.0958109125494957, Critic_loss = 0.016804879531264305\n",
      "Episode 14. Training Carried out at step = 720. Actorloss = 0.09451191127300262, Critic_loss = 0.005383361130952835\n",
      "Episode 14. Training Carried out at step = 730. Actorloss = 0.09435771405696869, Critic_loss = 0.005295869894325733\n",
      "Episode 14. Training Carried out at step = 740. Actorloss = 0.09431561827659607, Critic_loss = 0.017181871458888054\n",
      "Episode 14. Training Carried out at step = 750. Actorloss = 0.09434336423873901, Critic_loss = 0.025016218423843384\n",
      "Episode 14. Training Carried out at step = 760. Actorloss = 0.09433788061141968, Critic_loss = 0.02921478822827339\n",
      "Episode 14. Training Carried out at step = 770. Actorloss = 0.09477575123310089, Critic_loss = 0.020654279738664627\n",
      "Episode 14. Training Carried out at step = 780. Actorloss = 0.09506019949913025, Critic_loss = 0.00926361046731472\n",
      "Episode 14. Training Carried out at step = 790. Actorloss = 0.09510880708694458, Critic_loss = 0.01253598928451538\n",
      "Episode 14. Training Carried out at step = 800. Actorloss = 0.09497010707855225, Critic_loss = 0.04698141664266586\n",
      "Episode 14. Training Carried out at step = 810. Actorloss = 0.09526892751455307, Critic_loss = 0.09581653773784637\n",
      "Episode 14. Training Carried out at step = 820. Actorloss = 0.09880182892084122, Critic_loss = 0.6435603499412537\n",
      "Episode 14. Training Carried out at step = 830. Actorloss = 0.09689582139253616, Critic_loss = 0.36775410175323486\n",
      "Episode 14. Training Carried out at step = 840. Actorloss = 0.09622283279895782, Critic_loss = 0.213008850812912\n",
      "Episode 14. Training Carried out at step = 850. Actorloss = 0.0948181226849556, Critic_loss = 0.03083735890686512\n",
      "Episode 14. Training Carried out at step = 860. Actorloss = 0.09501094371080399, Critic_loss = 0.014500349760055542\n",
      "Episode 14. Training Carried out at step = 870. Actorloss = 0.0957598090171814, Critic_loss = 0.0037379320710897446\n",
      "Episode 14. Training Carried out at step = 880. Actorloss = 0.09526415169239044, Critic_loss = 0.0027081898879259825\n",
      "Episode 14. Training Carried out at step = 890. Actorloss = 0.09550141543149948, Critic_loss = 0.0029857316985726357\n",
      "Episode 14. Training Carried out at step = 900. Actorloss = 0.09561536461114883, Critic_loss = 0.003802409628406167\n",
      "Episode 14. Training Carried out at step = 910. Actorloss = 0.0958186686038971, Critic_loss = 0.006707276217639446\n",
      "Episode 14. Training Carried out at step = 920. Actorloss = 0.096170574426651, Critic_loss = 0.031032172963023186\n",
      "Episode 14. Training Carried out at step = 930. Actorloss = 0.09660868346691132, Critic_loss = 0.06523368507623672\n",
      "Episode 14. Training Carried out at step = 940. Actorloss = 0.09660933911800385, Critic_loss = 0.048071496188640594\n",
      "Episode 14. Training Carried out at step = 950. Actorloss = 0.0992944985628128, Critic_loss = 0.18092583119869232\n",
      "Episode 14. Training Carried out at step = 960. Actorloss = 0.09924522787332535, Critic_loss = 0.29556164145469666\n",
      "Episode 14. Training Carried out at step = 970. Actorloss = 0.09601567685604095, Critic_loss = 1.0407007932662964\n",
      "Episode 14. Training Carried out at step = 980. Actorloss = 0.09869683533906937, Critic_loss = 0.5969441533088684\n",
      "Episode 14. Training Carried out at step = 990. Actorloss = 0.09840982407331467, Critic_loss = 0.32320019602775574\n",
      "Episode 14. Training Carried out at step = 1000. Actorloss = 0.09765683114528656, Critic_loss = 0.16833578050136566\n",
      "Episode 14. Model checkpoint is saved \n",
      "\n",
      "Episode 14. tf.summary -> Tensorboard is updated \n",
      "\n",
      "Episode 15. Training Carried out at step = 10. Actorloss = 0.11218955367803574, Critic_loss = 0.17660821974277496\n",
      "Episode 15. Training Carried out at step = 20. Actorloss = 0.11147711426019669, Critic_loss = 0.19702808558940887\n",
      "Episode 15. Training Carried out at step = 30. Actorloss = 0.12435714900493622, Critic_loss = 1.1758114099502563\n",
      "Episode 15. Training Carried out at step = 40. Actorloss = 0.11301235854625702, Critic_loss = 0.3154211640357971\n",
      "Episode 15. Training Carried out at step = 50. Actorloss = 0.13454772531986237, Critic_loss = 3.8195383548736572\n",
      "Episode 15. Training Carried out at step = 60. Actorloss = 0.1547866016626358, Critic_loss = 5.8481316566467285\n",
      "Episode 15. Training Carried out at step = 70. Actorloss = 0.11405003070831299, Critic_loss = 1.1836711168289185\n",
      "Episode 15. Training Carried out at step = 80. Actorloss = 0.11370522528886795, Critic_loss = 0.3212886154651642\n",
      "Episode 15. Training Carried out at step = 90. Actorloss = 0.112616628408432, Critic_loss = 0.25010496377944946\n",
      "Episode 15. Training Carried out at step = 100. Actorloss = 0.1159677654504776, Critic_loss = 0.12374627590179443\n",
      "Episode 15. Training Carried out at step = 110. Actorloss = 0.12474613636732101, Critic_loss = 0.5181381702423096\n",
      "Episode 15. Training Carried out at step = 120. Actorloss = 0.11242275685071945, Critic_loss = 1.9268560409545898\n",
      "Episode 15. Training Carried out at step = 130. Actorloss = 0.11519443243741989, Critic_loss = 0.3302649259567261\n",
      "Episode 15. Training Carried out at step = 140. Actorloss = 0.1198459267616272, Critic_loss = 0.12436407059431076\n",
      "Episode 15. Training Carried out at step = 150. Actorloss = 0.13419978320598602, Critic_loss = 0.8397557735443115\n",
      "Episode 15. Training Carried out at step = 160. Actorloss = 0.12355010956525803, Critic_loss = 1.774192452430725\n",
      "Episode 15. Training Carried out at step = 170. Actorloss = 0.13212281465530396, Critic_loss = 1.510060429573059\n",
      "Episode 15. Training Carried out at step = 180. Actorloss = 0.14543646574020386, Critic_loss = 0.1431431919336319\n",
      "Episode 15. Training Carried out at step = 190. Actorloss = 0.14623308181762695, Critic_loss = 0.07103906571865082\n",
      "Episode 15. Training Carried out at step = 200. Actorloss = 0.16207954287528992, Critic_loss = 0.009866544976830482\n",
      "Episode 15. Training Carried out at step = 210. Actorloss = 0.16843192279338837, Critic_loss = 0.0177626796066761\n",
      "Episode 15. Training Carried out at step = 220. Actorloss = 0.1845155507326126, Critic_loss = 0.012910911813378334\n",
      "Episode 15. Training Carried out at step = 230. Actorloss = 0.1995105892419815, Critic_loss = 0.07666026055812836\n",
      "Episode 15. Training Carried out at step = 240. Actorloss = 0.15768209099769592, Critic_loss = 0.04484403133392334\n",
      "Episode 15. Training Carried out at step = 250. Actorloss = 0.15513794124126434, Critic_loss = 0.08837871253490448\n",
      "Episode 15. Training Carried out at step = 260. Actorloss = 0.13688784837722778, Critic_loss = 0.19335952401161194\n",
      "Episode 15. Training Carried out at step = 270. Actorloss = 0.12589800357818604, Critic_loss = 0.28452199697494507\n",
      "Episode 15. Training Carried out at step = 280. Actorloss = 0.12071677297353745, Critic_loss = 0.3635163903236389\n",
      "Episode 15. Training Carried out at step = 290. Actorloss = 0.11795275658369064, Critic_loss = 0.4365523159503937\n",
      "Episode 15. Training Carried out at step = 300. Actorloss = 0.11551836878061295, Critic_loss = 0.5269433856010437\n",
      "Episode 15. Training Carried out at step = 310. Actorloss = 0.11255807429552078, Critic_loss = 0.6450528502464294\n",
      "Episode 15. Training Carried out at step = 320. Actorloss = 0.1095520630478859, Critic_loss = 0.9442445635795593\n",
      "Episode 15. Training Carried out at step = 330. Actorloss = 0.10990002006292343, Critic_loss = 1.8663299083709717\n",
      "Episode 15. Training Carried out at step = 340. Actorloss = 0.11958690732717514, Critic_loss = 2.008017063140869\n",
      "Episode 15. Training Carried out at step = 350. Actorloss = 0.11740825325250626, Critic_loss = 0.234561949968338\n",
      "Episode 15. Training Carried out at step = 360. Actorloss = 0.11013218760490417, Critic_loss = 0.09542509913444519\n",
      "Episode 15. Training Carried out at step = 370. Actorloss = 0.11048542708158493, Critic_loss = 0.08192591369152069\n",
      "Episode 15. Training Carried out at step = 380. Actorloss = 0.11051979660987854, Critic_loss = 0.05525645613670349\n",
      "Episode 15. Training Carried out at step = 390. Actorloss = 0.1103699803352356, Critic_loss = 0.02914554439485073\n",
      "Episode 15. Training Carried out at step = 400. Actorloss = 0.11167144030332565, Critic_loss = 0.035613052546978\n",
      "Episode 15. Training Carried out at step = 410. Actorloss = 0.11211840063333511, Critic_loss = 0.032856788486242294\n",
      "Episode 15. Training Carried out at step = 420. Actorloss = 0.111275315284729, Critic_loss = 0.22419382631778717\n",
      "Episode 15. Training Carried out at step = 430. Actorloss = 0.10996570438146591, Critic_loss = 0.4128521978855133\n",
      "Episode 15. Training Carried out at step = 440. Actorloss = 0.11004747450351715, Critic_loss = 0.41646015644073486\n",
      "Episode 15. Training Carried out at step = 450. Actorloss = 0.11143850535154343, Critic_loss = 0.7758490443229675\n",
      "Episode 15. Training Carried out at step = 460. Actorloss = 0.11386699974536896, Critic_loss = 1.0631165504455566\n",
      "Episode 15. Training Carried out at step = 470. Actorloss = 0.1297890543937683, Critic_loss = 0.05247848108410835\n",
      "Episode 15. Training Carried out at step = 480. Actorloss = 0.121833436191082, Critic_loss = 0.0902644619345665\n",
      "Episode 15. Training Carried out at step = 490. Actorloss = 0.11261717975139618, Critic_loss = 0.01930776983499527\n",
      "Episode 15. Training Carried out at step = 500. Actorloss = 0.11182035505771637, Critic_loss = 0.12206865847110748\n",
      "Episode 15. Training Carried out at step = 510. Actorloss = 0.11952974647283554, Critic_loss = 0.6786940693855286\n",
      "Episode 15. Training Carried out at step = 520. Actorloss = 0.11997456103563309, Critic_loss = 1.4399746656417847\n",
      "Episode 15. Training Carried out at step = 530. Actorloss = 0.1237020492553711, Critic_loss = 1.2697306871414185\n",
      "Episode 15. Training Carried out at step = 540. Actorloss = 0.13202713429927826, Critic_loss = 1.354578971862793\n",
      "Episode 15. Training Carried out at step = 550. Actorloss = 0.13566797971725464, Critic_loss = 1.3071198463439941\n",
      "Episode 15. Training Carried out at step = 560. Actorloss = 0.11093994230031967, Critic_loss = 0.6347300410270691\n",
      "Episode 15. Training Carried out at step = 570. Actorloss = 0.1103118285536766, Critic_loss = 0.47858673334121704\n",
      "Episode 15. Training Carried out at step = 580. Actorloss = 0.11058883368968964, Critic_loss = 0.36986711621284485\n",
      "Episode 15. Training Carried out at step = 590. Actorloss = 0.11106985062360764, Critic_loss = 0.27734795212745667\n",
      "Episode 15. Training Carried out at step = 600. Actorloss = 0.1115003302693367, Critic_loss = 0.18583634495735168\n",
      "Episode 15. Training Carried out at step = 610. Actorloss = 0.11214232444763184, Critic_loss = 0.10883631557226181\n",
      "Episode 15. Training Carried out at step = 620. Actorloss = 0.1132553294301033, Critic_loss = 0.04934925213456154\n",
      "Episode 15. Training Carried out at step = 630. Actorloss = 0.11453641951084137, Critic_loss = 0.008687861263751984\n",
      "Episode 15. Training Carried out at step = 640. Actorloss = 0.11485995352268219, Critic_loss = 0.03859129548072815\n",
      "Episode 15. Training Carried out at step = 650. Actorloss = 0.1140948235988617, Critic_loss = 0.02032146044075489\n",
      "Episode 15. Training Carried out at step = 660. Actorloss = 0.11696998029947281, Critic_loss = 0.10020360350608826\n",
      "Episode 15. Training Carried out at step = 670. Actorloss = 0.11768552660942078, Critic_loss = 0.08022397756576538\n",
      "Episode 15. Training Carried out at step = 680. Actorloss = 0.11839443445205688, Critic_loss = 0.11638053506612778\n",
      "Episode 15. Training Carried out at step = 690. Actorloss = 0.11799048632383347, Critic_loss = 0.07416331768035889\n",
      "Episode 15. Training Carried out at step = 700. Actorloss = 0.12012329697608948, Critic_loss = 0.06391379237174988\n",
      "Episode 15. Training Carried out at step = 710. Actorloss = 0.12112297862768173, Critic_loss = 0.025771496817469597\n",
      "Episode 15. Training Carried out at step = 720. Actorloss = 0.12026368081569672, Critic_loss = 0.006485944613814354\n",
      "Episode 15. Training Carried out at step = 730. Actorloss = 0.11876081675291061, Critic_loss = 0.043540894985198975\n",
      "Episode 15. Training Carried out at step = 740. Actorloss = 0.11734675616025925, Critic_loss = 0.3265285789966583\n",
      "Episode 15. Training Carried out at step = 750. Actorloss = 0.11818186193704605, Critic_loss = 0.4138427972793579\n",
      "Episode 15. Training Carried out at step = 760. Actorloss = 0.11904624849557877, Critic_loss = 0.49369016289711\n",
      "Episode 15. Training Carried out at step = 770. Actorloss = 0.12482638657093048, Critic_loss = 1.1012072563171387\n",
      "Episode 15. Training Carried out at step = 780. Actorloss = 0.11920060217380524, Critic_loss = 0.22424116730690002\n",
      "Episode 15. Training Carried out at step = 790. Actorloss = 0.1178753599524498, Critic_loss = 0.07579401880502701\n",
      "Episode 15. Training Carried out at step = 800. Actorloss = 0.11786605417728424, Critic_loss = 0.025651585310697556\n",
      "Episode 15. Training Carried out at step = 810. Actorloss = 0.12225696444511414, Critic_loss = 0.15080119669437408\n",
      "Episode 15. Training Carried out at step = 820. Actorloss = 0.126566544175148, Critic_loss = 0.49584466218948364\n",
      "Episode 15. Training Carried out at step = 830. Actorloss = 0.13651378452777863, Critic_loss = 1.695171594619751\n",
      "Episode 15. Training Carried out at step = 840. Actorloss = 0.1344912201166153, Critic_loss = 1.8294432163238525\n",
      "Episode 15. Training Carried out at step = 850. Actorloss = 0.11741013079881668, Critic_loss = 0.35918575525283813\n",
      "Episode 15. Training Carried out at step = 860. Actorloss = 0.1144910678267479, Critic_loss = 0.5900346636772156\n",
      "Episode 15. Training Carried out at step = 870. Actorloss = 0.12253837287425995, Critic_loss = 0.16046471893787384\n",
      "Episode 15. Training Carried out at step = 880. Actorloss = 0.12485573440790176, Critic_loss = 0.012998194433748722\n",
      "Episode 15. Training Carried out at step = 890. Actorloss = 0.12543542683124542, Critic_loss = 0.005558244418352842\n",
      "Episode 15. Training Carried out at step = 900. Actorloss = 0.12580834329128265, Critic_loss = 0.002973355120047927\n",
      "Episode 15. Training Carried out at step = 910. Actorloss = 0.1263159215450287, Critic_loss = 0.0032579766120761633\n",
      "Episode 15. Training Carried out at step = 920. Actorloss = 0.12708625197410583, Critic_loss = 0.004491528961807489\n",
      "Episode 15. Training Carried out at step = 930. Actorloss = 0.12777860462665558, Critic_loss = 0.004956400953233242\n",
      "Episode 15. Training Carried out at step = 940. Actorloss = 0.12837976217269897, Critic_loss = 0.004574805498123169\n",
      "Episode 15. Training Carried out at step = 950. Actorloss = 0.12883692979812622, Critic_loss = 0.0035133312921971083\n",
      "Episode 15. Training Carried out at step = 960. Actorloss = 0.12714236974716187, Critic_loss = 0.03612181544303894\n",
      "Episode 15. Training Carried out at step = 970. Actorloss = 0.12480980157852173, Critic_loss = 0.0373292900621891\n",
      "Episode 15. Training Carried out at step = 980. Actorloss = 0.12352451682090759, Critic_loss = 0.0586467906832695\n",
      "Episode 15. Training Carried out at step = 990. Actorloss = 0.1385500729084015, Critic_loss = 0.06860651075839996\n",
      "Episode 15. Training Carried out at step = 1000. Actorloss = 0.13997116684913635, Critic_loss = 0.0840580016374588\n",
      "Episode 16. Training Carried out at step = 10. Actorloss = 0.14003317058086395, Critic_loss = 1.692408800125122\n",
      "Episode 16. Training Carried out at step = 20. Actorloss = 0.15487122535705566, Critic_loss = 1.9980796575546265\n",
      "Episode 16. Training Carried out at step = 30. Actorloss = 0.18915390968322754, Critic_loss = 0.5420856475830078\n",
      "Episode 16. Training Carried out at step = 40. Actorloss = 0.1517770141363144, Critic_loss = 0.990700900554657\n",
      "Episode 16. Training Carried out at step = 50. Actorloss = 0.18291734158992767, Critic_loss = 1.0136204957962036\n",
      "Episode 16. Training Carried out at step = 60. Actorloss = 0.1414971500635147, Critic_loss = 2.5331690311431885\n",
      "Episode 16. Training Carried out at step = 70. Actorloss = 0.13961699604988098, Critic_loss = 2.671143054962158\n",
      "Episode 16. Training Carried out at step = 80. Actorloss = 0.14120320975780487, Critic_loss = 1.2740007638931274\n",
      "Episode 16. Training Carried out at step = 90. Actorloss = 0.14102637767791748, Critic_loss = 1.1050677299499512\n",
      "Episode 16. Training Carried out at step = 100. Actorloss = 0.14289584755897522, Critic_loss = 0.9126806259155273\n",
      "Episode 16. Training Carried out at step = 110. Actorloss = 0.14290641248226166, Critic_loss = 0.8187260031700134\n",
      "Episode 16. Training Carried out at step = 120. Actorloss = 0.14316479861736298, Critic_loss = 0.364486426115036\n",
      "Episode 16. Training Carried out at step = 130. Actorloss = 0.1449459046125412, Critic_loss = 0.27125081419944763\n",
      "Episode 16. Training Carried out at step = 140. Actorloss = 0.14502841234207153, Critic_loss = 0.438612163066864\n",
      "Episode 16. Training Carried out at step = 150. Actorloss = 0.1419030874967575, Critic_loss = 0.3669116497039795\n",
      "Episode 16. Training Carried out at step = 160. Actorloss = 0.14202813804149628, Critic_loss = 0.1993182748556137\n",
      "Episode 16. Training Carried out at step = 170. Actorloss = 0.14243733882904053, Critic_loss = 0.09721405804157257\n",
      "Episode 16. Training Carried out at step = 180. Actorloss = 0.1504831314086914, Critic_loss = 0.10933090001344681\n",
      "Episode 16. Training Carried out at step = 190. Actorloss = 0.14322412014007568, Critic_loss = 0.015641294419765472\n",
      "Episode 16. Training Carried out at step = 200. Actorloss = 0.14369304478168488, Critic_loss = 0.008549869991838932\n",
      "Episode 16. Training Carried out at step = 210. Actorloss = 0.14407911896705627, Critic_loss = 0.006566157564520836\n",
      "Episode 16. Training Carried out at step = 220. Actorloss = 0.139897882938385, Critic_loss = 0.08025716990232468\n",
      "Episode 16. Training Carried out at step = 230. Actorloss = 0.1407221406698227, Critic_loss = 0.14033491909503937\n",
      "Episode 16. Training Carried out at step = 240. Actorloss = 0.1413974016904831, Critic_loss = 0.14062947034835815\n",
      "Episode 16. Training Carried out at step = 250. Actorloss = 0.14154058694839478, Critic_loss = 0.12408721446990967\n",
      "Episode 16. Training Carried out at step = 260. Actorloss = 0.14197541773319244, Critic_loss = 0.10464435815811157\n",
      "Episode 16. Training Carried out at step = 270. Actorloss = 0.14249679446220398, Critic_loss = 0.09041403234004974\n",
      "Episode 16. Training Carried out at step = 280. Actorloss = 0.1428406685590744, Critic_loss = 0.12159514427185059\n",
      "Episode 16. Training Carried out at step = 290. Actorloss = 0.14295971393585205, Critic_loss = 0.1850930154323578\n",
      "Episode 16. Training Carried out at step = 300. Actorloss = 0.14546391367912292, Critic_loss = 0.17922481894493103\n",
      "Episode 16. Training Carried out at step = 310. Actorloss = 0.14867736399173737, Critic_loss = 0.2591789662837982\n",
      "Episode 16. Training Carried out at step = 320. Actorloss = 0.1491110473871231, Critic_loss = 0.34338635206222534\n",
      "Episode 16. Training Carried out at step = 330. Actorloss = 0.14434802532196045, Critic_loss = 0.7356445789337158\n",
      "Episode 16. Training Carried out at step = 340. Actorloss = 0.1431093066930771, Critic_loss = 0.954860270023346\n",
      "Episode 16. Training Carried out at step = 350. Actorloss = 0.14856161177158356, Critic_loss = 0.33061307668685913\n",
      "Episode 16. Training Carried out at step = 360. Actorloss = 0.14908024668693542, Critic_loss = 0.32139670848846436\n",
      "Episode 16. Training Carried out at step = 370. Actorloss = 0.14611755311489105, Critic_loss = 0.17813697457313538\n",
      "Episode 16. Training Carried out at step = 380. Actorloss = 0.14512158930301666, Critic_loss = 0.03324287757277489\n",
      "Episode 16. Training Carried out at step = 390. Actorloss = 0.14609581232070923, Critic_loss = 0.01773707941174507\n",
      "Episode 16. Training Carried out at step = 400. Actorloss = 0.14743559062480927, Critic_loss = 0.010392991825938225\n",
      "Episode 16. Training Carried out at step = 410. Actorloss = 0.1459306925535202, Critic_loss = 0.055032458156347275\n",
      "Episode 16. Training Carried out at step = 420. Actorloss = 0.14651067554950714, Critic_loss = 0.08861777931451797\n",
      "Episode 16. Training Carried out at step = 430. Actorloss = 0.14866869151592255, Critic_loss = 0.09654015302658081\n",
      "Episode 16. Training Carried out at step = 440. Actorloss = 0.15225400030612946, Critic_loss = 0.09672725945711136\n",
      "Episode 16. Training Carried out at step = 450. Actorloss = 0.1531420648097992, Critic_loss = 0.0650356262922287\n",
      "Episode 16. Training Carried out at step = 460. Actorloss = 0.1530730426311493, Critic_loss = 0.05163395032286644\n",
      "Episode 16. Training Carried out at step = 470. Actorloss = 0.15301097929477692, Critic_loss = 0.062006279826164246\n",
      "Episode 16. Training Carried out at step = 480. Actorloss = 0.15275098383426666, Critic_loss = 0.11985644698143005\n",
      "Episode 16. Training Carried out at step = 490. Actorloss = 0.15245991945266724, Critic_loss = 0.16475625336170197\n",
      "Episode 16. Training Carried out at step = 500. Actorloss = 0.16115541756153107, Critic_loss = 0.5491126775741577\n",
      "Episode 16. Training Carried out at step = 510. Actorloss = 0.16285622119903564, Critic_loss = 1.0254915952682495\n",
      "Episode 16. Training Carried out at step = 520. Actorloss = 0.15679658949375153, Critic_loss = 0.38273271918296814\n",
      "Episode 16. Training Carried out at step = 530. Actorloss = 0.15213201940059662, Critic_loss = 0.25144627690315247\n",
      "Episode 16. Training Carried out at step = 540. Actorloss = 0.14982783794403076, Critic_loss = 0.0943702682852745\n",
      "Episode 16. Training Carried out at step = 550. Actorloss = 0.14983679354190826, Critic_loss = 0.032836250960826874\n",
      "Episode 16. Training Carried out at step = 560. Actorloss = 0.15271982550621033, Critic_loss = 0.008719815872609615\n",
      "Episode 16. Training Carried out at step = 570. Actorloss = 0.15263569355010986, Critic_loss = 0.004205686971545219\n",
      "Episode 16. Training Carried out at step = 580. Actorloss = 0.15307725965976715, Critic_loss = 0.0024671300780028105\n",
      "Episode 16. Training Carried out at step = 590. Actorloss = 0.15368656814098358, Critic_loss = 0.0070755137130618095\n",
      "Episode 16. Training Carried out at step = 600. Actorloss = 0.1543019860982895, Critic_loss = 0.020040635019540787\n",
      "Episode 16. Training Carried out at step = 610. Actorloss = 0.15535655617713928, Critic_loss = 0.04615188017487526\n",
      "Episode 16. Training Carried out at step = 620. Actorloss = 0.15646114945411682, Critic_loss = 0.011696342378854752\n",
      "Episode 16. Training Carried out at step = 630. Actorloss = 0.15674248337745667, Critic_loss = 0.060153715312480927\n",
      "Episode 16. Training Carried out at step = 640. Actorloss = 0.16693733632564545, Critic_loss = 0.0923168882727623\n",
      "Episode 16. Training Carried out at step = 650. Actorloss = 0.16365201771259308, Critic_loss = 0.15728026628494263\n",
      "Episode 16. Training Carried out at step = 660. Actorloss = 0.14725136756896973, Critic_loss = 0.9395294785499573\n",
      "Episode 16. Training Carried out at step = 670. Actorloss = 0.1587880551815033, Critic_loss = 0.1606084555387497\n",
      "Episode 16. Training Carried out at step = 680. Actorloss = 0.15979494154453278, Critic_loss = 0.0291989017277956\n",
      "Episode 16. Training Carried out at step = 690. Actorloss = 0.15620340406894684, Critic_loss = 0.12082663923501968\n",
      "Episode 16. Training Carried out at step = 700. Actorloss = 0.1603856235742569, Critic_loss = 0.23670972883701324\n",
      "Episode 16. Training Carried out at step = 710. Actorloss = 0.15943795442581177, Critic_loss = 0.2997535169124603\n",
      "Episode 16. Training Carried out at step = 720. Actorloss = 0.16143271327018738, Critic_loss = 0.3178105652332306\n",
      "Episode 16. Training Carried out at step = 730. Actorloss = 0.16014567017555237, Critic_loss = 0.29772093892097473\n",
      "Episode 16. Training Carried out at step = 740. Actorloss = 0.1612987220287323, Critic_loss = 0.2544570863246918\n",
      "Episode 16. Training Carried out at step = 750. Actorloss = 0.1609787791967392, Critic_loss = 0.18780209124088287\n",
      "Episode 16. Training Carried out at step = 760. Actorloss = 0.16158759593963623, Critic_loss = 0.12400881201028824\n",
      "Episode 16. Training Carried out at step = 770. Actorloss = 0.1669485718011856, Critic_loss = 0.06053352355957031\n",
      "Episode 16. Training Carried out at step = 780. Actorloss = 0.16745497286319733, Critic_loss = 0.026081571355462074\n",
      "Episode 16. Training Carried out at step = 790. Actorloss = 0.16831623017787933, Critic_loss = 0.018212633207440376\n",
      "Episode 16. Training Carried out at step = 800. Actorloss = 0.1671130210161209, Critic_loss = 0.05375388637185097\n",
      "Episode 16. Training Carried out at step = 810. Actorloss = 0.16914477944374084, Critic_loss = 0.12232498824596405\n",
      "Episode 16. Training Carried out at step = 820. Actorloss = 0.18759311735630035, Critic_loss = 0.8197630643844604\n",
      "Episode 16. Training Carried out at step = 830. Actorloss = 0.16967655718326569, Critic_loss = 0.20745019614696503\n",
      "Episode 16. Training Carried out at step = 840. Actorloss = 0.16712501645088196, Critic_loss = 0.22947359085083008\n",
      "Episode 16. Training Carried out at step = 850. Actorloss = 0.15390554070472717, Critic_loss = 0.027441121637821198\n",
      "Episode 16. Training Carried out at step = 860. Actorloss = 0.1461358666419983, Critic_loss = 0.04229454696178436\n",
      "Episode 16. Training Carried out at step = 870. Actorloss = 0.11495746672153473, Critic_loss = 0.13874836266040802\n",
      "Episode 16. Training Carried out at step = 880. Actorloss = 0.11098562926054001, Critic_loss = 0.1243148148059845\n",
      "Episode 16. Training Carried out at step = 890. Actorloss = 0.10978776216506958, Critic_loss = 0.08807789534330368\n",
      "Episode 16. Training Carried out at step = 900. Actorloss = 0.11175679415464401, Critic_loss = 0.03384223207831383\n",
      "Episode 16. Training Carried out at step = 910. Actorloss = 0.11265743523836136, Critic_loss = 0.0024446488823741674\n",
      "Episode 16. Training Carried out at step = 920. Actorloss = 0.11201629042625427, Critic_loss = 0.04121541231870651\n",
      "Episode 16. Training Carried out at step = 930. Actorloss = 0.11738710105419159, Critic_loss = 0.11186046153306961\n",
      "Episode 16. Training Carried out at step = 940. Actorloss = 0.11934616416692734, Critic_loss = 0.11818400025367737\n",
      "Episode 16. Training Carried out at step = 950. Actorloss = 0.11997223645448685, Critic_loss = 0.06891607493162155\n",
      "Episode 16. Training Carried out at step = 960. Actorloss = 0.12181544303894043, Critic_loss = 0.02358863689005375\n",
      "Episode 16. Training Carried out at step = 970. Actorloss = 0.12505632638931274, Critic_loss = 0.0042029982432723045\n",
      "Episode 16. Training Carried out at step = 980. Actorloss = 0.12611986696720123, Critic_loss = 0.009913972578942776\n",
      "Episode 16. Training Carried out at step = 990. Actorloss = 0.12687362730503082, Critic_loss = 0.01826871931552887\n",
      "Episode 16. Training Carried out at step = 1000. Actorloss = 0.12603241205215454, Critic_loss = 0.020126037299633026\n",
      "Episode 17. Training Carried out at step = 10. Actorloss = 0.11342363804578781, Critic_loss = 0.3564274311065674\n",
      "Episode 17. Training Carried out at step = 20. Actorloss = 0.11261924356222153, Critic_loss = 0.5417943000793457\n",
      "Episode 17. Training Carried out at step = 30. Actorloss = 0.15453195571899414, Critic_loss = 1.0506538152694702\n",
      "Episode 17. Training Carried out at step = 40. Actorloss = 0.12479355931282043, Critic_loss = 1.0792629718780518\n",
      "Episode 17. Training Carried out at step = 50. Actorloss = 0.11851366609334946, Critic_loss = 3.242339849472046\n",
      "Episode 17. Training Carried out at step = 60. Actorloss = 0.07365354895591736, Critic_loss = 0.6827207207679749\n",
      "Episode 17. Training Carried out at step = 70. Actorloss = 0.06525298207998276, Critic_loss = 0.02876386232674122\n",
      "Episode 17. Training Carried out at step = 80. Actorloss = 0.06339853256940842, Critic_loss = 0.007951642386615276\n",
      "Episode 17. Training Carried out at step = 90. Actorloss = 0.062031958252191544, Critic_loss = 0.011783099733293056\n",
      "Episode 17. Training Carried out at step = 100. Actorloss = 0.0606965497136116, Critic_loss = 0.014944139868021011\n",
      "Episode 17. Training Carried out at step = 110. Actorloss = 0.0595582015812397, Critic_loss = 0.013687445782124996\n",
      "Episode 17. Training Carried out at step = 120. Actorloss = 0.058533232659101486, Critic_loss = 0.011097578331828117\n",
      "Episode 17. Training Carried out at step = 130. Actorloss = 0.05757616460323334, Critic_loss = 0.008377013728022575\n",
      "Episode 17. Training Carried out at step = 140. Actorloss = 0.05663781613111496, Critic_loss = 0.006017136387526989\n",
      "Episode 17. Training Carried out at step = 150. Actorloss = 0.055698659271001816, Critic_loss = 0.004502078052610159\n",
      "Episode 17. Training Carried out at step = 160. Actorloss = 0.054763246327638626, Critic_loss = 0.004045525100082159\n",
      "Episode 17. Training Carried out at step = 170. Actorloss = 0.05383306369185448, Critic_loss = 0.004509285092353821\n",
      "Episode 17. Training Carried out at step = 180. Actorloss = 0.052902448922395706, Critic_loss = 0.005554941017180681\n",
      "Episode 17. Training Carried out at step = 190. Actorloss = 0.051975712180137634, Critic_loss = 0.006643736734986305\n",
      "Episode 17. Training Carried out at step = 200. Actorloss = 0.05101538449525833, Critic_loss = 0.00731211481615901\n",
      "Episode 17. Training Carried out at step = 210. Actorloss = 0.0500568225979805, Critic_loss = 0.007441952358931303\n",
      "Episode 17. Training Carried out at step = 220. Actorloss = 0.05806944891810417, Critic_loss = 0.007742289453744888\n",
      "Episode 17. Training Carried out at step = 230. Actorloss = 0.06257287412881851, Critic_loss = 0.010557972826063633\n",
      "Episode 17. Training Carried out at step = 240. Actorloss = 0.06234986335039139, Critic_loss = 0.009512430056929588\n",
      "Episode 17. Training Carried out at step = 250. Actorloss = 0.061409562826156616, Critic_loss = 0.007691119331866503\n",
      "Episode 17. Training Carried out at step = 260. Actorloss = 0.06038602069020271, Critic_loss = 0.006127670407295227\n",
      "Episode 17. Training Carried out at step = 270. Actorloss = 0.06639999896287918, Critic_loss = 0.006572382524609566\n",
      "Episode 17. Training Carried out at step = 280. Actorloss = 0.07016576826572418, Critic_loss = 0.005961428862065077\n",
      "Episode 17. Training Carried out at step = 290. Actorloss = 0.06739603728055954, Critic_loss = 0.0055673131719231606\n",
      "Episode 17. Training Carried out at step = 300. Actorloss = 0.06912821531295776, Critic_loss = 0.005849574226886034\n",
      "Episode 17. Training Carried out at step = 310. Actorloss = 0.06572789698839188, Critic_loss = 0.007332412526011467\n",
      "Episode 17. Training Carried out at step = 320. Actorloss = 0.06570835411548615, Critic_loss = 0.007885774597525597\n",
      "Episode 17. Training Carried out at step = 330. Actorloss = 0.06545034795999527, Critic_loss = 0.007789434865117073\n",
      "Episode 17. Training Carried out at step = 340. Actorloss = 0.06541018187999725, Critic_loss = 0.007111818064004183\n",
      "Episode 17. Training Carried out at step = 350. Actorloss = 0.06727562099695206, Critic_loss = 0.005927660036832094\n",
      "Episode 17. Training Carried out at step = 360. Actorloss = 0.06464638561010361, Critic_loss = 0.005502163432538509\n",
      "Episode 17. Training Carried out at step = 370. Actorloss = 0.0663490816950798, Critic_loss = 0.005085095763206482\n",
      "Episode 17. Training Carried out at step = 380. Actorloss = 0.06407175958156586, Critic_loss = 0.004962712060660124\n",
      "Episode 17. Training Carried out at step = 390. Actorloss = 0.06525339931249619, Critic_loss = 0.005343243479728699\n",
      "Episode 17. Training Carried out at step = 400. Actorloss = 0.0675831288099289, Critic_loss = 0.0062853251583874226\n",
      "Episode 17. Training Carried out at step = 410. Actorloss = 0.06460998207330704, Critic_loss = 0.0065318564884364605\n",
      "Episode 17. Training Carried out at step = 420. Actorloss = 0.05858222395181656, Critic_loss = 0.005331818014383316\n",
      "Episode 17. Training Carried out at step = 430. Actorloss = 0.055793989449739456, Critic_loss = 0.004494757391512394\n",
      "Episode 17. Training Carried out at step = 440. Actorloss = 0.05562925711274147, Critic_loss = 0.004448626656085253\n",
      "Episode 17. Training Carried out at step = 450. Actorloss = 0.06100064143538475, Critic_loss = 0.004793954081833363\n",
      "Episode 17. Training Carried out at step = 460. Actorloss = 0.06587985157966614, Critic_loss = 0.004934524651616812\n",
      "Episode 17. Training Carried out at step = 470. Actorloss = 0.06700169295072556, Critic_loss = 0.004739715717732906\n",
      "Episode 17. Training Carried out at step = 480. Actorloss = 0.07010601460933685, Critic_loss = 0.004687785170972347\n",
      "Episode 17. Training Carried out at step = 490. Actorloss = 0.06612411886453629, Critic_loss = 0.004617744125425816\n",
      "Episode 17. Training Carried out at step = 500. Actorloss = 0.06324561685323715, Critic_loss = 0.004557163454592228\n",
      "Episode 17. Training Carried out at step = 510. Actorloss = 0.06856320053339005, Critic_loss = 0.004579890985041857\n",
      "Episode 17. Training Carried out at step = 520. Actorloss = 0.0611848346889019, Critic_loss = 0.0045689367689192295\n",
      "Episode 17. Training Carried out at step = 530. Actorloss = 0.0695711076259613, Critic_loss = 0.004596626851707697\n",
      "Episode 17. Training Carried out at step = 540. Actorloss = 0.06693527102470398, Critic_loss = 0.0045267255045473576\n",
      "Episode 17. Training Carried out at step = 550. Actorloss = 0.06933996826410294, Critic_loss = 0.0045121805742383\n",
      "Episode 17. Training Carried out at step = 560. Actorloss = 0.06786497682332993, Critic_loss = 0.004385035950690508\n",
      "Episode 17. Training Carried out at step = 570. Actorloss = 0.06878862529993057, Critic_loss = 0.004404401406645775\n",
      "Episode 17. Training Carried out at step = 580. Actorloss = 0.07093321532011032, Critic_loss = 0.004341070540249348\n",
      "Episode 17. Training Carried out at step = 590. Actorloss = 0.067882239818573, Critic_loss = 0.004308851435780525\n",
      "Episode 17. Training Carried out at step = 600. Actorloss = 0.06821650266647339, Critic_loss = 0.004182132426649332\n",
      "Episode 17. Training Carried out at step = 610. Actorloss = 0.071409672498703, Critic_loss = 0.004231143742799759\n",
      "Episode 17. Training Carried out at step = 620. Actorloss = 0.06704336404800415, Critic_loss = 0.004079662729054689\n",
      "Episode 17. Training Carried out at step = 630. Actorloss = 0.07241509854793549, Critic_loss = 0.004035957157611847\n",
      "Episode 17. Training Carried out at step = 640. Actorloss = 0.07112394273281097, Critic_loss = 0.003958581015467644\n",
      "Episode 17. Training Carried out at step = 650. Actorloss = 0.07326770573854446, Critic_loss = 0.00408483250066638\n",
      "Episode 17. Training Carried out at step = 660. Actorloss = 0.06835878640413284, Critic_loss = 0.00399536918848753\n",
      "Episode 17. Training Carried out at step = 670. Actorloss = 0.07049403339624405, Critic_loss = 0.0039972481317818165\n",
      "Episode 17. Training Carried out at step = 680. Actorloss = 0.06968902051448822, Critic_loss = 0.003908838145434856\n",
      "Episode 17. Training Carried out at step = 690. Actorloss = 0.07182757556438446, Critic_loss = 0.003926429897546768\n",
      "Episode 17. Training Carried out at step = 700. Actorloss = 0.07070229202508926, Critic_loss = 0.0038306161295622587\n",
      "Episode 17. Training Carried out at step = 710. Actorloss = 0.07016266137361526, Critic_loss = 0.0037857117131352425\n",
      "Episode 17. Training Carried out at step = 720. Actorloss = 0.07067050039768219, Critic_loss = 0.0036848231684416533\n",
      "Episode 17. Training Carried out at step = 730. Actorloss = 0.06737300008535385, Critic_loss = 0.003613807028159499\n",
      "Episode 17. Training Carried out at step = 740. Actorloss = 0.0694989264011383, Critic_loss = 0.0036742608062922955\n",
      "Episode 17. Training Carried out at step = 750. Actorloss = 0.06775236129760742, Critic_loss = 0.0036203665658831596\n",
      "Episode 17. Training Carried out at step = 760. Actorloss = 0.06498660147190094, Critic_loss = 0.0035622462164610624\n",
      "Episode 17. Training Carried out at step = 770. Actorloss = 0.06594820320606232, Critic_loss = 0.0035455687902867794\n",
      "Episode 17. Training Carried out at step = 780. Actorloss = 0.0660676583647728, Critic_loss = 0.003515011165291071\n",
      "Episode 17. Training Carried out at step = 790. Actorloss = 0.06341123580932617, Critic_loss = 0.0034148863051086664\n",
      "Episode 17. Training Carried out at step = 800. Actorloss = 0.06326360255479813, Critic_loss = 0.0034372524823993444\n",
      "Episode 17. Training Carried out at step = 810. Actorloss = 0.06211027503013611, Critic_loss = 0.003380594775080681\n",
      "Episode 17. Training Carried out at step = 820. Actorloss = 0.061962369829416275, Critic_loss = 0.0033746936824172735\n",
      "Episode 17. Training Carried out at step = 830. Actorloss = 0.062392279505729675, Critic_loss = 0.0033652714919298887\n",
      "Episode 17. Training Carried out at step = 840. Actorloss = 0.06022513657808304, Critic_loss = 0.003337847301736474\n",
      "Episode 17. Training Carried out at step = 850. Actorloss = 0.06092394143342972, Critic_loss = 0.0033103525638580322\n",
      "Episode 17. Training Carried out at step = 860. Actorloss = 0.059086889028549194, Critic_loss = 0.0032860839273780584\n",
      "Episode 17. Training Carried out at step = 870. Actorloss = 0.05841415375471115, Critic_loss = 0.0032655964605510235\n",
      "Episode 17. Training Carried out at step = 880. Actorloss = 0.057932741940021515, Critic_loss = 0.00324602541513741\n",
      "Episode 17. Training Carried out at step = 890. Actorloss = 0.05803496390581131, Critic_loss = 0.003225509077310562\n",
      "Episode 17. Training Carried out at step = 900. Actorloss = 0.059493958950042725, Critic_loss = 0.0032193048391491175\n",
      "Episode 17. Training Carried out at step = 910. Actorloss = 0.06090417504310608, Critic_loss = 0.00320693152025342\n",
      "Episode 17. Training Carried out at step = 920. Actorloss = 0.07536179572343826, Critic_loss = 0.0034198311623185873\n",
      "Episode 17. Training Carried out at step = 930. Actorloss = 0.0757061243057251, Critic_loss = 0.003365503391250968\n",
      "Episode 17. Training Carried out at step = 940. Actorloss = 0.07593480497598648, Critic_loss = 0.0033139451406896114\n",
      "Episode 17. Training Carried out at step = 950. Actorloss = 0.07613015919923782, Critic_loss = 0.003267092164605856\n",
      "Episode 17. Training Carried out at step = 960. Actorloss = 0.07632552087306976, Critic_loss = 0.003226415952667594\n",
      "Episode 17. Training Carried out at step = 970. Actorloss = 0.07405373454093933, Critic_loss = 0.0031779182609170675\n",
      "Episode 17. Training Carried out at step = 980. Actorloss = 0.07932000607252121, Critic_loss = 0.0031695363577455282\n",
      "Episode 17. Training Carried out at step = 990. Actorloss = 0.07477553933858871, Critic_loss = 0.003162472741678357\n",
      "Episode 17. Training Carried out at step = 1000. Actorloss = 0.07718491554260254, Critic_loss = 0.0031348690390586853\n",
      "Episode 18. Training Carried out at step = 10. Actorloss = 0.11633569747209549, Critic_loss = 0.25847578048706055\n",
      "Episode 18. Training Carried out at step = 20. Actorloss = 0.10414782166481018, Critic_loss = 1.321426272392273\n",
      "Episode 18. Training Carried out at step = 30. Actorloss = 0.08156608790159225, Critic_loss = 4.604007244110107\n",
      "Episode 18. Training Carried out at step = 40. Actorloss = 0.10201410949230194, Critic_loss = 4.047006607055664\n",
      "Episode 18. Training Carried out at step = 50. Actorloss = 0.07994558662176132, Critic_loss = 1.2303348779678345\n",
      "Episode 18. Training Carried out at step = 60. Actorloss = 0.05410798266530037, Critic_loss = 0.07136034220457077\n",
      "Episode 18. Training Carried out at step = 70. Actorloss = 0.04858454689383507, Critic_loss = 0.029103703796863556\n",
      "Episode 18. Training Carried out at step = 80. Actorloss = 0.05381232872605324, Critic_loss = 0.01918010599911213\n",
      "Episode 18. Training Carried out at step = 90. Actorloss = 0.05499236285686493, Critic_loss = 0.0036648069508373737\n",
      "Episode 18. Training Carried out at step = 100. Actorloss = 0.04933147504925728, Critic_loss = 0.0059919944033026695\n",
      "Episode 18. Training Carried out at step = 110. Actorloss = 0.04822976887226105, Critic_loss = 0.010293942876160145\n",
      "Episode 18. Training Carried out at step = 120. Actorloss = 0.04833168908953667, Critic_loss = 0.013486336916685104\n",
      "Episode 18. Training Carried out at step = 130. Actorloss = 0.04875880479812622, Critic_loss = 0.014588483609259129\n",
      "Episode 18. Training Carried out at step = 140. Actorloss = 0.049296606332063675, Critic_loss = 0.013252783566713333\n",
      "Episode 18. Training Carried out at step = 150. Actorloss = 0.049000658094882965, Critic_loss = 0.010999953374266624\n",
      "Episode 18. Training Carried out at step = 160. Actorloss = 0.048996664583683014, Critic_loss = 0.007988479919731617\n",
      "Episode 18. Training Carried out at step = 170. Actorloss = 0.04927212372422218, Critic_loss = 0.005577195435762405\n",
      "Episode 18. Training Carried out at step = 180. Actorloss = 0.04904569685459137, Critic_loss = 0.004636647179722786\n",
      "Episode 18. Training Carried out at step = 190. Actorloss = 0.048826850950717926, Critic_loss = 0.005666443612426519\n",
      "Episode 18. Training Carried out at step = 200. Actorloss = 0.04871911555528641, Critic_loss = 0.007571882102638483\n",
      "Episode 18. Training Carried out at step = 210. Actorloss = 0.0486937090754509, Critic_loss = 0.008178213611245155\n",
      "Episode 18. Training Carried out at step = 220. Actorloss = 0.04866533726453781, Critic_loss = 0.007597383577376604\n",
      "Episode 18. Training Carried out at step = 230. Actorloss = 0.04863584414124489, Critic_loss = 0.006451294291764498\n",
      "Episode 18. Training Carried out at step = 240. Actorloss = 0.04991497844457626, Critic_loss = 0.005572329740971327\n",
      "Episode 18. Training Carried out at step = 250. Actorloss = 0.05482218787074089, Critic_loss = 0.005398495122790337\n",
      "Episode 18. Training Carried out at step = 260. Actorloss = 0.054801978170871735, Critic_loss = 0.004885879345238209\n",
      "Episode 18. Training Carried out at step = 270. Actorloss = 0.05479038879275322, Critic_loss = 0.0047845738008618355\n",
      "Episode 18. Training Carried out at step = 280. Actorloss = 0.054783016443252563, Critic_loss = 0.004933535121381283\n",
      "Episode 18. Training Carried out at step = 290. Actorloss = 0.05477628856897354, Critic_loss = 0.005163371097296476\n",
      "Episode 18. Training Carried out at step = 300. Actorloss = 0.05476900562644005, Critic_loss = 0.005352730862796307\n",
      "Episode 18. Training Carried out at step = 310. Actorloss = 0.054759763181209564, Critic_loss = 0.0054398090578615665\n",
      "Episode 18. Training Carried out at step = 320. Actorloss = 0.05474970489740372, Critic_loss = 0.005414582788944244\n",
      "Episode 18. Training Carried out at step = 330. Actorloss = 0.05473649874329567, Critic_loss = 0.005297186318784952\n",
      "Episode 18. Training Carried out at step = 340. Actorloss = 0.05471901595592499, Critic_loss = 0.005124181974679232\n",
      "Episode 18. Training Carried out at step = 350. Actorloss = 0.05469579994678497, Critic_loss = 0.004940877202898264\n",
      "Episode 18. Training Carried out at step = 360. Actorloss = 0.054666128009557724, Critic_loss = 0.004785267170518637\n",
      "Episode 18. Training Carried out at step = 370. Actorloss = 0.05463093891739845, Critic_loss = 0.004669038113206625\n",
      "Episode 18. Training Carried out at step = 380. Actorloss = 0.054591186344623566, Critic_loss = 0.004600392188876867\n",
      "Episode 18. Training Carried out at step = 390. Actorloss = 0.054545607417821884, Critic_loss = 0.004579294007271528\n",
      "Episode 18. Training Carried out at step = 400. Actorloss = 0.05449485406279564, Critic_loss = 0.004590015392750502\n",
      "Episode 18. Training Carried out at step = 410. Actorloss = 0.05443960055708885, Critic_loss = 0.004613320808857679\n",
      "Episode 18. Training Carried out at step = 420. Actorloss = 0.05438299849629402, Critic_loss = 0.004627487156540155\n",
      "Episode 18. Training Carried out at step = 430. Actorloss = 0.05432481691241264, Critic_loss = 0.004624132066965103\n",
      "Episode 18. Training Carried out at step = 440. Actorloss = 0.0542651005089283, Critic_loss = 0.004597139079123735\n",
      "Episode 18. Training Carried out at step = 450. Actorloss = 0.05420490726828575, Critic_loss = 0.004546810872852802\n",
      "Episode 18. Training Carried out at step = 460. Actorloss = 0.05414736643433571, Critic_loss = 0.004479757510125637\n",
      "Episode 18. Training Carried out at step = 470. Actorloss = 0.054090507328510284, Critic_loss = 0.0044056870974600315\n",
      "Episode 18. Training Carried out at step = 480. Actorloss = 0.05403465777635574, Critic_loss = 0.00433368468657136\n",
      "Episode 18. Training Carried out at step = 490. Actorloss = 0.05398015305399895, Critic_loss = 0.004271825309842825\n",
      "Episode 18. Training Carried out at step = 500. Actorloss = 0.05392704904079437, Critic_loss = 0.004223398864269257\n",
      "Episode 18. Training Carried out at step = 510. Actorloss = 0.053875572979450226, Critic_loss = 0.004188469145447016\n",
      "Episode 18. Training Carried out at step = 520. Actorloss = 0.0538272000849247, Critic_loss = 0.004163791425526142\n",
      "Episode 18. Training Carried out at step = 530. Actorloss = 0.053780391812324524, Critic_loss = 0.004144744016230106\n",
      "Episode 18. Training Carried out at step = 540. Actorloss = 0.05322378873825073, Critic_loss = 0.00412594061344862\n",
      "Episode 18. Training Carried out at step = 550. Actorloss = 0.053080540150403976, Critic_loss = 0.004104975610971451\n",
      "Episode 18. Training Carried out at step = 560. Actorloss = 0.0530373677611351, Critic_loss = 0.004076898563653231\n",
      "Episode 18. Training Carried out at step = 570. Actorloss = 0.05299071967601776, Critic_loss = 0.004041593987494707\n",
      "Episode 18. Training Carried out at step = 580. Actorloss = 0.05294285714626312, Critic_loss = 0.004001372493803501\n",
      "Episode 18. Training Carried out at step = 590. Actorloss = 0.05289354547858238, Critic_loss = 0.003959377761930227\n",
      "Episode 18. Training Carried out at step = 600. Actorloss = 0.052842527627944946, Critic_loss = 0.003918591421097517\n",
      "Episode 18. Training Carried out at step = 610. Actorloss = 0.05244836211204529, Critic_loss = 0.0038827997632324696\n",
      "Episode 18. Training Carried out at step = 620. Actorloss = 0.05202147737145424, Critic_loss = 0.003852949943393469\n",
      "Episode 18. Training Carried out at step = 630. Actorloss = 0.05169438570737839, Critic_loss = 0.003823336446657777\n",
      "Episode 18. Training Carried out at step = 640. Actorloss = 0.05159858986735344, Critic_loss = 0.003797739278525114\n",
      "Episode 18. Training Carried out at step = 650. Actorloss = 0.05150163918733597, Critic_loss = 0.0037759519182145596\n",
      "Episode 18. Training Carried out at step = 660. Actorloss = 0.05140304937958717, Critic_loss = 0.0037559058982878923\n",
      "Episode 18. Training Carried out at step = 670. Actorloss = 0.05130362883210182, Critic_loss = 0.0037360666319727898\n",
      "Episode 18. Training Carried out at step = 680. Actorloss = 0.05120319873094559, Critic_loss = 0.003714795457199216\n",
      "Episode 18. Training Carried out at step = 690. Actorloss = 0.0511024035513401, Critic_loss = 0.0036920916754752398\n",
      "Episode 18. Training Carried out at step = 700. Actorloss = 0.05100191757082939, Critic_loss = 0.0036679478362202644\n",
      "Episode 18. Training Carried out at step = 710. Actorloss = 0.05090157315135002, Critic_loss = 0.0036426782608032227\n",
      "Episode 18. Training Carried out at step = 720. Actorloss = 0.050801344215869904, Critic_loss = 0.003616074798628688\n",
      "Episode 18. Training Carried out at step = 730. Actorloss = 0.050699181854724884, Critic_loss = 0.003591216169297695\n",
      "Episode 18. Training Carried out at step = 740. Actorloss = 0.05059608817100525, Critic_loss = 0.003566933562979102\n",
      "Episode 18. Training Carried out at step = 750. Actorloss = 0.049935482442379, Critic_loss = 0.0035464614629745483\n",
      "Episode 18. Training Carried out at step = 760. Actorloss = 0.04972707852721214, Critic_loss = 0.0035272566601634026\n",
      "Episode 18. Training Carried out at step = 770. Actorloss = 0.049618951976299286, Critic_loss = 0.003508654423058033\n",
      "Episode 18. Training Carried out at step = 780. Actorloss = 0.04950815066695213, Critic_loss = 0.00349023868329823\n",
      "Episode 18. Training Carried out at step = 790. Actorloss = 0.04939745366573334, Critic_loss = 0.0034717803355306387\n",
      "Episode 18. Training Carried out at step = 800. Actorloss = 0.04928663372993469, Critic_loss = 0.0034528677351772785\n",
      "Episode 18. Training Carried out at step = 810. Actorloss = 0.049175430089235306, Critic_loss = 0.0034333886578679085\n",
      "Episode 18. Training Carried out at step = 820. Actorloss = 0.04859454929828644, Critic_loss = 0.0034194074105471373\n",
      "Episode 18. Training Carried out at step = 830. Actorloss = 0.04838874191045761, Critic_loss = 0.0034005814231932163\n",
      "Episode 18. Training Carried out at step = 840. Actorloss = 0.048267755657434464, Critic_loss = 0.0033807391300797462\n",
      "Episode 18. Training Carried out at step = 850. Actorloss = 0.04814538359642029, Critic_loss = 0.0033620926551520824\n",
      "Episode 18. Training Carried out at step = 860. Actorloss = 0.04802229255437851, Critic_loss = 0.003344016382470727\n",
      "Episode 18. Training Carried out at step = 870. Actorloss = 0.047898534685373306, Critic_loss = 0.003327100770547986\n",
      "Episode 18. Training Carried out at step = 880. Actorloss = 0.04777412861585617, Critic_loss = 0.003310971660539508\n",
      "Episode 18. Training Carried out at step = 890. Actorloss = 0.04727156087756157, Critic_loss = 0.0032970141619443893\n",
      "Episode 18. Training Carried out at step = 900. Actorloss = 0.04706571251153946, Critic_loss = 0.0032818822655826807\n",
      "Episode 18. Training Carried out at step = 910. Actorloss = 0.046931710094213486, Critic_loss = 0.003266747109591961\n",
      "Episode 18. Training Carried out at step = 920. Actorloss = 0.04679723083972931, Critic_loss = 0.0032518156804144382\n",
      "Episode 18. Training Carried out at step = 930. Actorloss = 0.04666256532073021, Critic_loss = 0.003237165044993162\n",
      "Episode 18. Training Carried out at step = 940. Actorloss = 0.046527743339538574, Critic_loss = 0.0032222203444689512\n",
      "Episode 18. Training Carried out at step = 950. Actorloss = 0.04639289155602455, Critic_loss = 0.003207299392670393\n",
      "Episode 18. Training Carried out at step = 960. Actorloss = 0.04625803232192993, Critic_loss = 0.003192764474079013\n",
      "Episode 18. Training Carried out at step = 970. Actorloss = 0.046123214066028595, Critic_loss = 0.003178634447976947\n",
      "Episode 18. Training Carried out at step = 980. Actorloss = 0.04568856209516525, Critic_loss = 0.0031658587977290154\n",
      "Episode 18. Training Carried out at step = 990. Actorloss = 0.04548606649041176, Critic_loss = 0.003152735298499465\n",
      "Episode 18. Training Carried out at step = 1000. Actorloss = 0.045343801379203796, Critic_loss = 0.003139570588245988\n",
      "Episode 19. Training Carried out at step = 10. Actorloss = 0.11531616747379303, Critic_loss = 0.3966096043586731\n",
      "Episode 19. Training Carried out at step = 20. Actorloss = 0.1048101857304573, Critic_loss = 2.29885196685791\n",
      "Episode 19. Training Carried out at step = 30. Actorloss = 0.10016679018735886, Critic_loss = 8.399391174316406\n",
      "Episode 19. Training Carried out at step = 40. Actorloss = 0.08067378401756287, Critic_loss = 21.96590805053711\n",
      "Episode 19. Training Carried out at step = 50. Actorloss = 0.04736547917127609, Critic_loss = 20.4905948638916\n",
      "Episode 19. Training Carried out at step = 60. Actorloss = 0.04490470886230469, Critic_loss = 0.01639178954064846\n",
      "Episode 19. Training Carried out at step = 70. Actorloss = 0.04307527840137482, Critic_loss = 0.024448947980999947\n",
      "Episode 19. Training Carried out at step = 80. Actorloss = 0.04477815702557564, Critic_loss = 0.0320662297308445\n",
      "Episode 19. Training Carried out at step = 90. Actorloss = 0.040487147867679596, Critic_loss = 0.022620776668190956\n",
      "Episode 19. Training Carried out at step = 100. Actorloss = 0.03951840102672577, Critic_loss = 0.015266980044543743\n",
      "Episode 19. Training Carried out at step = 110. Actorloss = 0.03865597024559975, Critic_loss = 0.008057409897446632\n",
      "Episode 19. Training Carried out at step = 120. Actorloss = 0.03794809803366661, Critic_loss = 0.004562534857541323\n",
      "Episode 19. Training Carried out at step = 130. Actorloss = 0.03742942586541176, Critic_loss = 0.0032212266232818365\n",
      "Episode 19. Training Carried out at step = 140. Actorloss = 0.0369567908346653, Critic_loss = 0.0035009346902370453\n",
      "Episode 19. Training Carried out at step = 150. Actorloss = 0.0365556962788105, Critic_loss = 0.004755347967147827\n",
      "Episode 19. Training Carried out at step = 160. Actorloss = 0.03625667467713356, Critic_loss = 0.006412002258002758\n",
      "Episode 19. Training Carried out at step = 170. Actorloss = 0.036194898188114166, Critic_loss = 0.00804406963288784\n",
      "Episode 19. Training Carried out at step = 180. Actorloss = 0.038447342813014984, Critic_loss = 0.00888589397072792\n",
      "Episode 19. Training Carried out at step = 190. Actorloss = 0.03772949427366257, Critic_loss = 0.009374363347887993\n",
      "Episode 19. Training Carried out at step = 200. Actorloss = 0.04067642614245415, Critic_loss = 0.011293437331914902\n",
      "Episode 19. Training Carried out at step = 210. Actorloss = 0.036392632871866226, Critic_loss = 0.006062998436391354\n",
      "Episode 19. Training Carried out at step = 220. Actorloss = 0.05349234119057655, Critic_loss = 0.005949229933321476\n",
      "Episode 19. Training Carried out at step = 230. Actorloss = 0.05476478114724159, Critic_loss = 0.006342780310660601\n",
      "Episode 19. Training Carried out at step = 240. Actorloss = 0.054611992090940475, Critic_loss = 0.005857809446752071\n",
      "Episode 19. Training Carried out at step = 250. Actorloss = 0.05446385219693184, Critic_loss = 0.004928181413561106\n",
      "Episode 19. Training Carried out at step = 260. Actorloss = 0.054321691393852234, Critic_loss = 0.004010690376162529\n",
      "Episode 19. Training Carried out at step = 270. Actorloss = 0.05418409779667854, Critic_loss = 0.003424542723223567\n",
      "Episode 19. Training Carried out at step = 280. Actorloss = 0.054049111902713776, Critic_loss = 0.0031534237787127495\n",
      "Episode 19. Training Carried out at step = 290. Actorloss = 0.053918223828077316, Critic_loss = 0.0030903571750968695\n",
      "Episode 19. Training Carried out at step = 300. Actorloss = 0.05379088968038559, Critic_loss = 0.003125216346234083\n",
      "Episode 19. Training Carried out at step = 310. Actorloss = 0.053664833307266235, Critic_loss = 0.0031702984124422073\n",
      "Episode 19. Training Carried out at step = 320. Actorloss = 0.053539253771305084, Critic_loss = 0.003183850785717368\n",
      "Episode 19. Training Carried out at step = 330. Actorloss = 0.053413424640893936, Critic_loss = 0.003165786387398839\n",
      "Episode 19. Training Carried out at step = 340. Actorloss = 0.053286995738744736, Critic_loss = 0.0031213746406137943\n",
      "Episode 19. Training Carried out at step = 350. Actorloss = 0.05316305160522461, Critic_loss = 0.0030649458058178425\n",
      "Episode 19. Training Carried out at step = 360. Actorloss = 0.0530393123626709, Critic_loss = 0.0030081006698310375\n",
      "Episode 19. Training Carried out at step = 370. Actorloss = 0.05291670933365822, Critic_loss = 0.0029598025139421225\n",
      "Episode 19. Training Carried out at step = 380. Actorloss = 0.05279823765158653, Critic_loss = 0.0029288162477314472\n",
      "Episode 19. Training Carried out at step = 390. Actorloss = 0.05268577113747597, Critic_loss = 0.0029136387165635824\n",
      "Episode 19. Training Carried out at step = 400. Actorloss = 0.05258675292134285, Critic_loss = 0.0029117478989064693\n",
      "Episode 19. Training Carried out at step = 410. Actorloss = 0.05250944569706917, Critic_loss = 0.002915885066613555\n",
      "Episode 19. Training Carried out at step = 420. Actorloss = 0.05244409665465355, Critic_loss = 0.0029206150211393833\n",
      "Episode 19. Training Carried out at step = 430. Actorloss = 0.052388790994882584, Critic_loss = 0.0029213938396424055\n",
      "Episode 19. Training Carried out at step = 440. Actorloss = 0.052336353808641434, Critic_loss = 0.0029130473267287016\n",
      "Episode 19. Training Carried out at step = 450. Actorloss = 0.05228593572974205, Critic_loss = 0.00289915525354445\n",
      "Episode 19. Training Carried out at step = 460. Actorloss = 0.05223766714334488, Critic_loss = 0.0028798815328627825\n",
      "Episode 19. Training Carried out at step = 470. Actorloss = 0.05219171941280365, Critic_loss = 0.002856502076610923\n",
      "Episode 19. Training Carried out at step = 480. Actorloss = 0.052148159593343735, Critic_loss = 0.0028360190335661173\n",
      "Episode 19. Training Carried out at step = 490. Actorloss = 0.05210696533322334, Critic_loss = 0.002819439396262169\n",
      "Episode 19. Training Carried out at step = 500. Actorloss = 0.05206803232431412, Critic_loss = 0.002807131502777338\n",
      "Episode 19. Training Carried out at step = 510. Actorloss = 0.05203113332390785, Critic_loss = 0.0027990646194666624\n",
      "Episode 19. Training Carried out at step = 520. Actorloss = 0.05199600011110306, Critic_loss = 0.002794348867610097\n",
      "Episode 19. Training Carried out at step = 530. Actorloss = 0.051962342113256454, Critic_loss = 0.0027907502371817827\n",
      "Episode 19. Training Carried out at step = 540. Actorloss = 0.051929835230112076, Critic_loss = 0.0027870757039636374\n",
      "Episode 19. Training Carried out at step = 550. Actorloss = 0.05189789831638336, Critic_loss = 0.002781194867566228\n",
      "Episode 19. Training Carried out at step = 560. Actorloss = 0.05186590924859047, Critic_loss = 0.0027725016698241234\n",
      "Episode 19. Training Carried out at step = 570. Actorloss = 0.05183180049061775, Critic_loss = 0.0027623299974948168\n",
      "Episode 19. Training Carried out at step = 580. Actorloss = 0.05179379880428314, Critic_loss = 0.002750770654529333\n",
      "Episode 19. Training Carried out at step = 590. Actorloss = 0.05174955353140831, Critic_loss = 0.0027387982700020075\n",
      "Episode 19. Training Carried out at step = 600. Actorloss = 0.051696617156267166, Critic_loss = 0.002727230777963996\n",
      "Episode 19. Training Carried out at step = 610. Actorloss = 0.05163275822997093, Critic_loss = 0.002717423951253295\n",
      "Episode 19. Training Carried out at step = 620. Actorloss = 0.051560863852500916, Critic_loss = 0.002708863466978073\n",
      "Episode 19. Training Carried out at step = 630. Actorloss = 0.05148491635918617, Critic_loss = 0.0027016708627343178\n",
      "Episode 19. Training Carried out at step = 640. Actorloss = 0.05140780657529831, Critic_loss = 0.0026952812913805246\n",
      "Episode 19. Training Carried out at step = 650. Actorloss = 0.05133023485541344, Critic_loss = 0.0026895315386354923\n",
      "Episode 19. Training Carried out at step = 660. Actorloss = 0.0512520857155323, Critic_loss = 0.0026834968011826277\n",
      "Episode 19. Training Carried out at step = 670. Actorloss = 0.0511724054813385, Critic_loss = 0.002676947508007288\n",
      "Episode 19. Training Carried out at step = 680. Actorloss = 0.05108711123466492, Critic_loss = 0.0026698410511016846\n",
      "Episode 19. Training Carried out at step = 690. Actorloss = 0.05099109932780266, Critic_loss = 0.002661994891241193\n",
      "Episode 19. Training Carried out at step = 700. Actorloss = 0.05088445171713829, Critic_loss = 0.002654357347637415\n",
      "Episode 19. Training Carried out at step = 710. Actorloss = 0.05076957121491432, Critic_loss = 0.002646287437528372\n",
      "Episode 19. Training Carried out at step = 720. Actorloss = 0.05064467713236809, Critic_loss = 0.0026387071702629328\n",
      "Episode 19. Training Carried out at step = 730. Actorloss = 0.0505153127014637, Critic_loss = 0.002631267998367548\n",
      "Episode 19. Training Carried out at step = 740. Actorloss = 0.050385210663080215, Critic_loss = 0.0026243505999445915\n",
      "Episode 19. Training Carried out at step = 750. Actorloss = 0.05025502294301987, Critic_loss = 0.0026177577674388885\n",
      "Episode 19. Training Carried out at step = 760. Actorloss = 0.05012432858347893, Critic_loss = 0.0026113749481737614\n",
      "Episode 19. Training Carried out at step = 770. Actorloss = 0.04999320209026337, Critic_loss = 0.0026052275206893682\n",
      "Episode 19. Training Carried out at step = 780. Actorloss = 0.04986196756362915, Critic_loss = 0.0025991809088736773\n",
      "Episode 19. Training Carried out at step = 790. Actorloss = 0.04973073676228523, Critic_loss = 0.002593014156445861\n",
      "Episode 19. Training Carried out at step = 800. Actorloss = 0.04959946870803833, Critic_loss = 0.0025868448428809643\n",
      "Episode 19. Training Carried out at step = 810. Actorloss = 0.04946809634566307, Critic_loss = 0.0025802769232541323\n",
      "Episode 19. Training Carried out at step = 820. Actorloss = 0.04933658614754677, Critic_loss = 0.002573928562924266\n",
      "Episode 19. Training Carried out at step = 830. Actorloss = 0.04920487478375435, Critic_loss = 0.002567345043644309\n",
      "Episode 19. Training Carried out at step = 840. Actorloss = 0.04907296225428581, Critic_loss = 0.0025609773583710194\n",
      "Episode 19. Training Carried out at step = 850. Actorloss = 0.04894084110856056, Critic_loss = 0.002554790349677205\n",
      "Episode 19. Training Carried out at step = 860. Actorloss = 0.048808470368385315, Critic_loss = 0.0025486652739346027\n",
      "Episode 19. Training Carried out at step = 870. Actorloss = 0.048675935715436935, Critic_loss = 0.0025427406653761864\n",
      "Episode 19. Training Carried out at step = 880. Actorloss = 0.04854324460029602, Critic_loss = 0.0025369080249220133\n",
      "Episode 19. Training Carried out at step = 890. Actorloss = 0.04841041937470436, Critic_loss = 0.0025311545468866825\n",
      "Episode 19. Training Carried out at step = 900. Actorloss = 0.048277515918016434, Critic_loss = 0.002525357762351632\n",
      "Episode 19. Training Carried out at step = 910. Actorloss = 0.04814456030726433, Critic_loss = 0.0025196881033480167\n",
      "Episode 19. Training Carried out at step = 920. Actorloss = 0.04801161214709282, Critic_loss = 0.0025139646604657173\n",
      "Episode 19. Training Carried out at step = 930. Actorloss = 0.0478786900639534, Critic_loss = 0.0025082239881157875\n",
      "Episode 19. Training Carried out at step = 940. Actorloss = 0.04774583876132965, Critic_loss = 0.002502469578757882\n",
      "Episode 19. Training Carried out at step = 950. Actorloss = 0.04761306568980217, Critic_loss = 0.0024968429934233427\n",
      "Episode 19. Training Carried out at step = 960. Actorloss = 0.04748038575053215, Critic_loss = 0.002491214545443654\n",
      "Episode 19. Training Carried out at step = 970. Actorloss = 0.04734782129526138, Critic_loss = 0.002485685283318162\n",
      "Episode 19. Training Carried out at step = 980. Actorloss = 0.047215335071086884, Critic_loss = 0.0024802174884825945\n",
      "Episode 19. Training Carried out at step = 990. Actorloss = 0.04708296060562134, Critic_loss = 0.002474837237969041\n",
      "Episode 19. Training Carried out at step = 1000. Actorloss = 0.04695064574480057, Critic_loss = 0.002469463972374797\n",
      "Episode 19. Model checkpoint is saved \n",
      "\n",
      "Episode 19. tf.summary -> Tensorboard is updated \n",
      "\n",
      "Episode 20. Training Carried out at step = 10. Actorloss = 0.12114312499761581, Critic_loss = 0.233148455619812\n",
      "Episode 20. Training Carried out at step = 20. Actorloss = 0.09142855554819107, Critic_loss = 3.2850050926208496\n",
      "Episode 20. Training Carried out at step = 30. Actorloss = 0.07822328805923462, Critic_loss = 11.460201263427734\n",
      "Episode 20. Training Carried out at step = 40. Actorloss = 0.04222498834133148, Critic_loss = 26.776824951171875\n",
      "Episode 20. Training Carried out at step = 50. Actorloss = 0.054258812218904495, Critic_loss = 10.88512134552002\n",
      "Episode 20. Training Carried out at step = 60. Actorloss = 0.054969631135463715, Critic_loss = 0.025871330872178078\n",
      "Episode 20. Training Carried out at step = 70. Actorloss = 0.048535220324993134, Critic_loss = 0.05849463492631912\n",
      "Episode 20. Training Carried out at step = 80. Actorloss = 0.0462789349257946, Critic_loss = 0.050015151500701904\n",
      "Episode 20. Training Carried out at step = 90. Actorloss = 0.04404926300048828, Critic_loss = 0.026306258514523506\n",
      "Episode 20. Training Carried out at step = 100. Actorloss = 0.041805852204561234, Critic_loss = 0.013445224612951279\n",
      "Episode 20. Training Carried out at step = 110. Actorloss = 0.03953171893954277, Critic_loss = 0.009265519678592682\n",
      "Episode 20. Training Carried out at step = 120. Actorloss = 0.03732643648982048, Critic_loss = 0.005744730122387409\n",
      "Episode 20. Training Carried out at step = 130. Actorloss = 0.03538263216614723, Critic_loss = 0.0035437368787825108\n",
      "Episode 20. Training Carried out at step = 140. Actorloss = 0.033678486943244934, Critic_loss = 0.002833428094163537\n",
      "Episode 20. Training Carried out at step = 150. Actorloss = 0.032338034361600876, Critic_loss = 0.003322263713926077\n",
      "Episode 20. Training Carried out at step = 160. Actorloss = 0.03120054490864277, Critic_loss = 0.004543965682387352\n",
      "Episode 20. Training Carried out at step = 170. Actorloss = 0.03027433343231678, Critic_loss = 0.005953496787697077\n",
      "Episode 20. Training Carried out at step = 180. Actorloss = 0.029308587312698364, Critic_loss = 0.0071969348937273026\n",
      "Episode 20. Training Carried out at step = 190. Actorloss = 0.028247084468603134, Critic_loss = 0.007925711572170258\n",
      "Episode 20. Training Carried out at step = 200. Actorloss = 0.027279596775770187, Critic_loss = 0.007911778055131435\n",
      "Episode 20. Training Carried out at step = 210. Actorloss = 0.03661505877971649, Critic_loss = 0.005612732842564583\n",
      "Episode 20. Training Carried out at step = 220. Actorloss = 0.059705354273319244, Critic_loss = 0.028495213016867638\n",
      "Episode 20. Training Carried out at step = 230. Actorloss = 0.06851886957883835, Critic_loss = 0.02477431669831276\n",
      "Episode 20. Training Carried out at step = 240. Actorloss = 0.06822147220373154, Critic_loss = 0.007112197577953339\n",
      "Episode 20. Training Carried out at step = 250. Actorloss = 0.06792905926704407, Critic_loss = 0.002867770381271839\n",
      "Episode 20. Training Carried out at step = 260. Actorloss = 0.06764157116413116, Critic_loss = 0.004150325898081064\n",
      "Episode 20. Training Carried out at step = 270. Actorloss = 0.06735695153474808, Critic_loss = 0.005765818990767002\n",
      "Episode 20. Training Carried out at step = 280. Actorloss = 0.06707271188497543, Critic_loss = 0.0064716581255197525\n",
      "Episode 20. Training Carried out at step = 290. Actorloss = 0.06678614765405655, Critic_loss = 0.006359963212162256\n",
      "Episode 20. Training Carried out at step = 300. Actorloss = 0.06649487465620041, Critic_loss = 0.005806215573102236\n",
      "Episode 20. Training Carried out at step = 310. Actorloss = 0.06619688123464584, Critic_loss = 0.004995117429643869\n",
      "Episode 20. Training Carried out at step = 320. Actorloss = 0.06589072942733765, Critic_loss = 0.0041220239363610744\n",
      "Episode 20. Training Carried out at step = 330. Actorloss = 0.05220743641257286, Critic_loss = 0.003935962915420532\n",
      "Episode 20. Training Carried out at step = 340. Actorloss = 0.04842611774802208, Critic_loss = 0.0033031650818884373\n",
      "Episode 20. Training Carried out at step = 350. Actorloss = 0.04773138836026192, Critic_loss = 0.002856491832062602\n",
      "Episode 20. Training Carried out at step = 360. Actorloss = 0.046798836439847946, Critic_loss = 0.0027353884652256966\n",
      "Episode 20. Training Carried out at step = 370. Actorloss = 0.0456550195813179, Critic_loss = 0.002856559120118618\n",
      "Episode 20. Training Carried out at step = 380. Actorloss = 0.04415996000170708, Critic_loss = 0.003118108492344618\n",
      "Episode 20. Training Carried out at step = 390. Actorloss = 0.042110148817300797, Critic_loss = 0.0033897235989570618\n",
      "Episode 20. Training Carried out at step = 400. Actorloss = 0.03988751024007797, Critic_loss = 0.0036704568192362785\n",
      "Episode 20. Training Carried out at step = 410. Actorloss = 0.03875232860445976, Critic_loss = 0.0037914803251624107\n",
      "Episode 20. Training Carried out at step = 420. Actorloss = 0.03788711875677109, Critic_loss = 0.003702928777784109\n",
      "Episode 20. Training Carried out at step = 430. Actorloss = 0.037033937871456146, Critic_loss = 0.0034817096311599016\n",
      "Episode 20. Training Carried out at step = 440. Actorloss = 0.03619483858346939, Critic_loss = 0.0032061466481536627\n",
      "Episode 20. Training Carried out at step = 450. Actorloss = 0.03537425771355629, Critic_loss = 0.002945773536339402\n",
      "Episode 20. Training Carried out at step = 460. Actorloss = 0.03459792956709862, Critic_loss = 0.0027572093531489372\n",
      "Episode 20. Training Carried out at step = 470. Actorloss = 0.03383450210094452, Critic_loss = 0.002667056629434228\n",
      "Episode 20. Training Carried out at step = 480. Actorloss = 0.03307759761810303, Critic_loss = 0.002670965390279889\n",
      "Episode 20. Training Carried out at step = 490. Actorloss = 0.03232685849070549, Critic_loss = 0.0027428423054516315\n",
      "Episode 20. Training Carried out at step = 500. Actorloss = 0.0315958671271801, Critic_loss = 0.002843656577169895\n",
      "Episode 20. Training Carried out at step = 510. Actorloss = 0.030879175290465355, Critic_loss = 0.0029368081595748663\n",
      "Episode 20. Training Carried out at step = 520. Actorloss = 0.030165895819664, Critic_loss = 0.0029928237199783325\n",
      "Episode 20. Training Carried out at step = 530. Actorloss = 0.029454994946718216, Critic_loss = 0.002998142037540674\n",
      "Episode 20. Training Carried out at step = 540. Actorloss = 0.028745479881763458, Critic_loss = 0.0029563268180936575\n",
      "Episode 20. Training Carried out at step = 550. Actorloss = 0.03692799434065819, Critic_loss = 0.0028888490051031113\n",
      "Episode 20. Training Carried out at step = 560. Actorloss = 0.030281592160463333, Critic_loss = 0.0028944038785994053\n",
      "Episode 20. Training Carried out at step = 570. Actorloss = 0.029995808377861977, Critic_loss = 0.0027622480411082506\n",
      "Episode 20. Training Carried out at step = 580. Actorloss = 0.0297072846442461, Critic_loss = 0.0026616903487592936\n",
      "Episode 20. Training Carried out at step = 590. Actorloss = 0.0294333528727293, Critic_loss = 0.002604631008580327\n",
      "Episode 20. Training Carried out at step = 600. Actorloss = 0.029189594089984894, Critic_loss = 0.0025940602645277977\n",
      "Episode 20. Training Carried out at step = 610. Actorloss = 0.02896472066640854, Critic_loss = 0.002620811341330409\n",
      "Episode 20. Training Carried out at step = 620. Actorloss = 0.028737928718328476, Critic_loss = 0.0026681246235966682\n",
      "Episode 20. Training Carried out at step = 630. Actorloss = 0.02850998193025589, Critic_loss = 0.0027153724804520607\n",
      "Episode 20. Training Carried out at step = 640. Actorloss = 0.028281686827540398, Critic_loss = 0.002746904268860817\n",
      "Episode 20. Training Carried out at step = 650. Actorloss = 0.028061874210834503, Critic_loss = 0.0027717631310224533\n",
      "Episode 20. Training Carried out at step = 660. Actorloss = 0.027879711240530014, Critic_loss = 0.0027358303777873516\n",
      "Episode 20. Training Carried out at step = 670. Actorloss = 0.02771664969623089, Critic_loss = 0.0026977015659213066\n",
      "Episode 20. Training Carried out at step = 680. Actorloss = 0.027562426403164864, Critic_loss = 0.0026474136393517256\n",
      "Episode 20. Training Carried out at step = 690. Actorloss = 0.027449874207377434, Critic_loss = 0.002604820765554905\n",
      "Episode 20. Training Carried out at step = 700. Actorloss = 0.027355723083019257, Critic_loss = 0.0025675296783447266\n",
      "Episode 20. Training Carried out at step = 710. Actorloss = 0.027262287214398384, Critic_loss = 0.002543356968089938\n",
      "Episode 20. Training Carried out at step = 720. Actorloss = 0.027171527966856956, Critic_loss = 0.0025322490837424994\n",
      "Episode 20. Training Carried out at step = 730. Actorloss = 0.02708323672413826, Critic_loss = 0.0025321661960333586\n",
      "Episode 20. Training Carried out at step = 740. Actorloss = 0.02699706144630909, Critic_loss = 0.002537955529987812\n",
      "Episode 20. Training Carried out at step = 750. Actorloss = 0.026934392750263214, Critic_loss = 0.0025443932972848415\n",
      "Episode 20. Training Carried out at step = 760. Actorloss = 0.026876335963606834, Critic_loss = 0.002549227559939027\n",
      "Episode 20. Training Carried out at step = 770. Actorloss = 0.026842398568987846, Critic_loss = 0.0025465446524322033\n",
      "Episode 20. Training Carried out at step = 780. Actorloss = 0.02683616243302822, Critic_loss = 0.002536324318498373\n",
      "Episode 20. Training Carried out at step = 790. Actorloss = 0.026852136477828026, Critic_loss = 0.0025237533263862133\n",
      "Episode 20. Training Carried out at step = 800. Actorloss = 0.026867667213082314, Critic_loss = 0.0025099453050643206\n",
      "Episode 20. Training Carried out at step = 810. Actorloss = 0.03770070523023605, Critic_loss = 0.0024990979582071304\n",
      "Episode 20. Training Carried out at step = 820. Actorloss = 0.03740227222442627, Critic_loss = 0.0024911114014685154\n",
      "Episode 20. Training Carried out at step = 830. Actorloss = 0.037089381366968155, Critic_loss = 0.002484316471964121\n",
      "Episode 20. Training Carried out at step = 840. Actorloss = 0.036776553839445114, Critic_loss = 0.0024784267880022526\n",
      "Episode 20. Training Carried out at step = 850. Actorloss = 0.036463797092437744, Critic_loss = 0.0024738097563385963\n",
      "Episode 20. Training Carried out at step = 860. Actorloss = 0.03615116700530052, Critic_loss = 0.002469922648742795\n",
      "Episode 20. Training Carried out at step = 870. Actorloss = 0.03583869710564613, Critic_loss = 0.0024669584818184376\n",
      "Episode 20. Training Carried out at step = 880. Actorloss = 0.03552600368857384, Critic_loss = 0.002463803393766284\n",
      "Episode 20. Training Carried out at step = 890. Actorloss = 0.03521258383989334, Critic_loss = 0.0024597172159701586\n",
      "Episode 20. Training Carried out at step = 900. Actorloss = 0.03489717096090317, Critic_loss = 0.002455678768455982\n",
      "Episode 20. Training Carried out at step = 910. Actorloss = 0.034581635147333145, Critic_loss = 0.0024517327547073364\n",
      "Episode 20. Training Carried out at step = 920. Actorloss = 0.03426670283079147, Critic_loss = 0.0024463562294840813\n",
      "Episode 20. Training Carried out at step = 930. Actorloss = 0.033952441066503525, Critic_loss = 0.0024417871609330177\n",
      "Episode 20. Training Carried out at step = 940. Actorloss = 0.03363887965679169, Critic_loss = 0.0024369056336581707\n",
      "Episode 20. Training Carried out at step = 950. Actorloss = 0.033326055854558945, Critic_loss = 0.002432227833196521\n",
      "Episode 20. Training Carried out at step = 960. Actorloss = 0.03301393613219261, Critic_loss = 0.002428104169666767\n",
      "Episode 20. Training Carried out at step = 970. Actorloss = 0.03270250931382179, Critic_loss = 0.0024242389481514692\n",
      "Episode 20. Training Carried out at step = 980. Actorloss = 0.03239171952009201, Critic_loss = 0.0024206102825701237\n",
      "Episode 20. Training Carried out at step = 990. Actorloss = 0.032081522047519684, Critic_loss = 0.002417048206552863\n",
      "Episode 20. Training Carried out at step = 1000. Actorloss = 0.031771834939718246, Critic_loss = 0.002413516165688634\n",
      "Episode 21. Training Carried out at step = 10. Actorloss = 0.095237135887146, Critic_loss = 0.18980281054973602\n",
      "Episode 21. Training Carried out at step = 20. Actorloss = 0.07028279453516006, Critic_loss = 4.067329406738281\n",
      "Episode 21. Training Carried out at step = 30. Actorloss = 0.04859035089612007, Critic_loss = 12.350998878479004\n",
      "Episode 21. Training Carried out at step = 40. Actorloss = 0.027845893055200577, Critic_loss = 21.673606872558594\n",
      "Episode 21. Training Carried out at step = 50. Actorloss = 0.07179955393075943, Critic_loss = 5.753350734710693\n",
      "Episode 21. Training Carried out at step = 60. Actorloss = 0.07977979630231857, Critic_loss = 0.0731176808476448\n",
      "Episode 21. Training Carried out at step = 70. Actorloss = 0.0750134140253067, Critic_loss = 0.09654165059328079\n",
      "Episode 21. Training Carried out at step = 80. Actorloss = 0.07143483310937881, Critic_loss = 0.0917828381061554\n",
      "Episode 21. Training Carried out at step = 90. Actorloss = 0.06786122173070908, Critic_loss = 0.07489033788442612\n",
      "Episode 21. Training Carried out at step = 100. Actorloss = 0.06429235637187958, Critic_loss = 0.050986431539058685\n",
      "Episode 21. Training Carried out at step = 110. Actorloss = 0.06077774986624718, Critic_loss = 0.026688680052757263\n",
      "Episode 21. Training Carried out at step = 120. Actorloss = 0.057328686118125916, Critic_loss = 0.010757066309452057\n",
      "Episode 21. Training Carried out at step = 130. Actorloss = 0.05394420400261879, Critic_loss = 0.0038908314891159534\n",
      "Episode 21. Training Carried out at step = 140. Actorloss = 0.05062057077884674, Critic_loss = 0.002700086450204253\n",
      "Episode 21. Training Carried out at step = 150. Actorloss = 0.047348640859127045, Critic_loss = 0.005108325742185116\n",
      "Episode 21. Training Carried out at step = 160. Actorloss = 0.04413016140460968, Critic_loss = 0.009368427097797394\n",
      "Episode 21. Training Carried out at step = 170. Actorloss = 0.040915679186582565, Critic_loss = 0.013660942204296589\n",
      "Episode 21. Training Carried out at step = 180. Actorloss = 0.0376417376101017, Critic_loss = 0.01708780787885189\n",
      "Episode 21. Training Carried out at step = 190. Actorloss = 0.034387558698654175, Critic_loss = 0.018615668639540672\n",
      "Episode 21. Training Carried out at step = 200. Actorloss = 0.034868765622377396, Critic_loss = 0.01726575382053852\n",
      "Episode 21. Training Carried out at step = 210. Actorloss = 0.03942215070128441, Critic_loss = 0.013377215713262558\n",
      "Episode 21. Training Carried out at step = 220. Actorloss = 0.03195901960134506, Critic_loss = 0.011430876329541206\n",
      "Episode 21. Training Carried out at step = 230. Actorloss = 0.02432921901345253, Critic_loss = 0.008271913975477219\n",
      "Episode 21. Training Carried out at step = 240. Actorloss = 0.024074487388134003, Critic_loss = 0.004128267988562584\n",
      "Episode 21. Training Carried out at step = 250. Actorloss = 0.02380584366619587, Critic_loss = 0.002615835051983595\n",
      "Episode 21. Training Carried out at step = 260. Actorloss = 0.02352931536734104, Critic_loss = 0.0038869203999638557\n",
      "Episode 21. Training Carried out at step = 270. Actorloss = 0.023244302719831467, Critic_loss = 0.0063613783568143845\n",
      "Episode 21. Training Carried out at step = 280. Actorloss = 0.02295268326997757, Critic_loss = 0.007912712171673775\n",
      "Episode 21. Training Carried out at step = 290. Actorloss = 0.02265927754342556, Critic_loss = 0.007667820900678635\n",
      "Episode 21. Training Carried out at step = 300. Actorloss = 0.02236827462911606, Critic_loss = 0.0061810435727238655\n",
      "Episode 21. Training Carried out at step = 310. Actorloss = 0.022082742303609848, Critic_loss = 0.004469896666705608\n",
      "Episode 21. Training Carried out at step = 320. Actorloss = 0.021804500371217728, Critic_loss = 0.0031920592300593853\n",
      "Episode 21. Training Carried out at step = 330. Actorloss = 0.021533310413360596, Critic_loss = 0.0026184420567005873\n",
      "Episode 21. Training Carried out at step = 340. Actorloss = 0.021269332617521286, Critic_loss = 0.0026441558729857206\n",
      "Episode 21. Training Carried out at step = 350. Actorloss = 0.021011188626289368, Critic_loss = 0.00297750998288393\n",
      "Episode 21. Training Carried out at step = 360. Actorloss = 0.020755086094141006, Critic_loss = 0.0031729426700621843\n",
      "Episode 21. Training Carried out at step = 370. Actorloss = 0.02050216868519783, Critic_loss = 0.0032476549968123436\n",
      "Episode 21. Training Carried out at step = 380. Actorloss = 0.020250964909791946, Critic_loss = 0.0032444940879940987\n",
      "Episode 21. Training Carried out at step = 390. Actorloss = 0.020000141113996506, Critic_loss = 0.0031663968693464994\n",
      "Episode 21. Training Carried out at step = 400. Actorloss = 0.01974855177104473, Critic_loss = 0.0030379786621779203\n",
      "Episode 21. Training Carried out at step = 410. Actorloss = 0.019495250657200813, Critic_loss = 0.002873161807656288\n",
      "Episode 21. Training Carried out at step = 420. Actorloss = 0.019239623099565506, Critic_loss = 0.002724860329180956\n",
      "Episode 21. Training Carried out at step = 430. Actorloss = 0.01898771896958351, Critic_loss = 0.002608777955174446\n",
      "Episode 21. Training Carried out at step = 440. Actorloss = 0.018754441291093826, Critic_loss = 0.0025384237524122\n",
      "Episode 21. Training Carried out at step = 450. Actorloss = 0.0185202956199646, Critic_loss = 0.002515802625566721\n",
      "Episode 21. Training Carried out at step = 460. Actorloss = 0.018297890201210976, Critic_loss = 0.0025326842442154884\n",
      "Episode 21. Training Carried out at step = 470. Actorloss = 0.018088068813085556, Critic_loss = 0.0025761735159903765\n",
      "Episode 21. Training Carried out at step = 480. Actorloss = 0.017878418788313866, Critic_loss = 0.002622918225824833\n",
      "Episode 21. Training Carried out at step = 490. Actorloss = 0.01766808144748211, Critic_loss = 0.0026661334559321404\n",
      "Episode 21. Training Carried out at step = 500. Actorloss = 0.017458543181419373, Critic_loss = 0.002695186994969845\n",
      "Episode 21. Training Carried out at step = 510. Actorloss = 0.017264066264033318, Critic_loss = 0.0026929492596536875\n",
      "Episode 21. Training Carried out at step = 520. Actorloss = 0.017070909962058067, Critic_loss = 0.002676597097888589\n",
      "Episode 21. Training Carried out at step = 530. Actorloss = 0.01687966287136078, Critic_loss = 0.00263962778262794\n",
      "Episode 21. Training Carried out at step = 540. Actorloss = 0.01669984683394432, Critic_loss = 0.0025942667853087187\n",
      "Episode 21. Training Carried out at step = 550. Actorloss = 0.016543762758374214, Critic_loss = 0.002549360040575266\n",
      "Episode 21. Training Carried out at step = 560. Actorloss = 0.01641133427619934, Critic_loss = 0.002509427722543478\n",
      "Episode 21. Training Carried out at step = 570. Actorloss = 0.016289418563246727, Critic_loss = 0.002483611460775137\n",
      "Episode 21. Training Carried out at step = 580. Actorloss = 0.016183307394385338, Critic_loss = 0.0024680134374648333\n",
      "Episode 21. Training Carried out at step = 590. Actorloss = 0.016095764935016632, Critic_loss = 0.0024623216595500708\n",
      "Episode 21. Training Carried out at step = 600. Actorloss = 0.016010401770472527, Critic_loss = 0.0024651633575558662\n",
      "Episode 21. Training Carried out at step = 610. Actorloss = 0.015926919877529144, Critic_loss = 0.0024706050753593445\n",
      "Episode 21. Training Carried out at step = 620. Actorloss = 0.01584496535360813, Critic_loss = 0.0024767438881099224\n",
      "Episode 21. Training Carried out at step = 630. Actorloss = 0.01576418988406658, Critic_loss = 0.0024797171354293823\n",
      "Episode 21. Training Carried out at step = 640. Actorloss = 0.01568421721458435, Critic_loss = 0.0024797338992357254\n",
      "Episode 21. Training Carried out at step = 650. Actorloss = 0.015613281168043613, Critic_loss = 0.002471246989443898\n",
      "Episode 21. Training Carried out at step = 660. Actorloss = 0.01555358711630106, Critic_loss = 0.0024623475037515163\n",
      "Episode 21. Training Carried out at step = 670. Actorloss = 0.015493744052946568, Critic_loss = 0.0024509201757609844\n",
      "Episode 21. Training Carried out at step = 680. Actorloss = 0.015449663624167442, Critic_loss = 0.0024391189217567444\n",
      "Episode 21. Training Carried out at step = 690. Actorloss = 0.015409942716360092, Critic_loss = 0.0024296955671161413\n",
      "Episode 21. Training Carried out at step = 700. Actorloss = 0.015408086590468884, Critic_loss = 0.0024228161200881004\n",
      "Episode 21. Training Carried out at step = 710. Actorloss = 0.015406294725835323, Critic_loss = 0.0024182661436498165\n",
      "Episode 21. Training Carried out at step = 720. Actorloss = 0.015403667464852333, Critic_loss = 0.0024167471565306187\n",
      "Episode 21. Training Carried out at step = 730. Actorloss = 0.01540062204003334, Critic_loss = 0.002416654722765088\n",
      "Episode 21. Training Carried out at step = 740. Actorloss = 0.01539734099060297, Critic_loss = 0.0024176347069442272\n",
      "Episode 21. Training Carried out at step = 750. Actorloss = 0.015411514788866043, Critic_loss = 0.0024174796417355537\n",
      "Episode 21. Training Carried out at step = 760. Actorloss = 0.015429974533617496, Critic_loss = 0.0024157154839485884\n",
      "Episode 21. Training Carried out at step = 770. Actorloss = 0.015450751408934593, Critic_loss = 0.002412940841168165\n",
      "Episode 21. Training Carried out at step = 780. Actorloss = 0.015493991784751415, Critic_loss = 0.00241031008772552\n",
      "Episode 21. Training Carried out at step = 790. Actorloss = 0.015537722036242485, Critic_loss = 0.002403195248916745\n",
      "Episode 21. Training Carried out at step = 800. Actorloss = 0.015600800514221191, Critic_loss = 0.002397884614765644\n",
      "Episode 21. Training Carried out at step = 810. Actorloss = 0.015681836754083633, Critic_loss = 0.002391526708379388\n",
      "Episode 21. Training Carried out at step = 820. Actorloss = 0.0157785601913929, Critic_loss = 0.0023846349213272333\n",
      "Episode 21. Training Carried out at step = 830. Actorloss = 0.015899205580353737, Critic_loss = 0.0023787198588252068\n",
      "Episode 21. Training Carried out at step = 840. Actorloss = 0.016041774302721024, Critic_loss = 0.002373228082433343\n",
      "Episode 21. Training Carried out at step = 850. Actorloss = 0.016191400587558746, Critic_loss = 0.002369093708693981\n",
      "Episode 21. Training Carried out at step = 860. Actorloss = 0.016364526003599167, Critic_loss = 0.0023653386160731316\n",
      "Episode 21. Training Carried out at step = 870. Actorloss = 0.016550853848457336, Critic_loss = 0.002361989812925458\n",
      "Episode 21. Training Carried out at step = 880. Actorloss = 0.016776956617832184, Critic_loss = 0.002358851255849004\n",
      "Episode 21. Training Carried out at step = 890. Actorloss = 0.017006246373057365, Critic_loss = 0.0023559462279081345\n",
      "Episode 21. Training Carried out at step = 900. Actorloss = 0.017235109582543373, Critic_loss = 0.002352361101657152\n",
      "Episode 21. Training Carried out at step = 910. Actorloss = 0.01746354065835476, Critic_loss = 0.002348961541429162\n",
      "Episode 21. Training Carried out at step = 920. Actorloss = 0.017691560089588165, Critic_loss = 0.0023457498755306005\n",
      "Episode 21. Training Carried out at step = 930. Actorloss = 0.017919188365340233, Critic_loss = 0.00234194565564394\n",
      "Episode 21. Training Carried out at step = 940. Actorloss = 0.018146447837352753, Critic_loss = 0.002338355639949441\n",
      "Episode 21. Training Carried out at step = 950. Actorloss = 0.018373500555753708, Critic_loss = 0.002335013123229146\n",
      "Episode 21. Training Carried out at step = 960. Actorloss = 0.018653789535164833, Critic_loss = 0.002331708325073123\n",
      "Episode 21. Training Carried out at step = 970. Actorloss = 0.018942929804325104, Critic_loss = 0.0023287441581487656\n",
      "Episode 21. Training Carried out at step = 980. Actorloss = 0.019229765981435776, Critic_loss = 0.0023259855806827545\n",
      "Episode 21. Training Carried out at step = 990. Actorloss = 0.019515786319971085, Critic_loss = 0.0023231152445077896\n",
      "Episode 21. Training Carried out at step = 1000. Actorloss = 0.019801197573542595, Critic_loss = 0.0023204057943075895\n",
      "Episode 22. Training Carried out at step = 10. Actorloss = 0.04787522181868553, Critic_loss = 0.11508157104253769\n",
      "Episode 22. Training Carried out at step = 20. Actorloss = 0.033264514058828354, Critic_loss = 2.4117681980133057\n",
      "Episode 22. Training Carried out at step = 30. Actorloss = 0.03220954164862633, Critic_loss = 9.05661392211914\n",
      "Episode 22. Training Carried out at step = 40. Actorloss = 0.04098315164446831, Critic_loss = 17.26115608215332\n",
      "Episode 22. Training Carried out at step = 50. Actorloss = 0.05165378004312515, Critic_loss = 4.1215996742248535\n",
      "Episode 22. Training Carried out at step = 60. Actorloss = 0.08286627382040024, Critic_loss = 0.13654908537864685\n",
      "Episode 22. Training Carried out at step = 70. Actorloss = 0.092191681265831, Critic_loss = 0.1384461373090744\n",
      "Episode 22. Training Carried out at step = 80. Actorloss = 0.08998265117406845, Critic_loss = 0.11459620296955109\n",
      "Episode 22. Training Carried out at step = 90. Actorloss = 0.08780176937580109, Critic_loss = 0.07906978577375412\n",
      "Episode 22. Training Carried out at step = 100. Actorloss = 0.0855695977807045, Critic_loss = 0.04301346838474274\n",
      "Episode 22. Training Carried out at step = 110. Actorloss = 0.08336903154850006, Critic_loss = 0.024746637791395187\n",
      "Episode 22. Training Carried out at step = 120. Actorloss = 0.08122739940881729, Critic_loss = 0.015037206932902336\n",
      "Episode 22. Training Carried out at step = 130. Actorloss = 0.07903210073709488, Critic_loss = 0.0076209548860788345\n",
      "Episode 22. Training Carried out at step = 140. Actorloss = 0.07654472440481186, Critic_loss = 0.0034628347493708134\n",
      "Episode 22. Training Carried out at step = 150. Actorloss = 0.07361315190792084, Critic_loss = 0.0025750286877155304\n",
      "Episode 22. Training Carried out at step = 160. Actorloss = 0.07049273699522018, Critic_loss = 0.004346635192632675\n",
      "Episode 22. Training Carried out at step = 170. Actorloss = 0.06742081791162491, Critic_loss = 0.0075611574575304985\n",
      "Episode 22. Training Carried out at step = 180. Actorloss = 0.06429433077573776, Critic_loss = 0.01121247187256813\n",
      "Episode 22. Training Carried out at step = 190. Actorloss = 0.061143387109041214, Critic_loss = 0.014145920053124428\n",
      "Episode 22. Training Carried out at step = 200. Actorloss = 0.06144138053059578, Critic_loss = 0.014978233724832535\n",
      "Episode 22. Training Carried out at step = 210. Actorloss = 0.0618707649409771, Critic_loss = 0.014286577701568604\n",
      "Episode 22. Training Carried out at step = 220. Actorloss = 0.030346324667334557, Critic_loss = 0.01896812953054905\n",
      "Episode 22. Training Carried out at step = 230. Actorloss = 0.027152247726917267, Critic_loss = 0.01606164127588272\n",
      "Episode 22. Training Carried out at step = 240. Actorloss = 0.027524437755346298, Critic_loss = 0.011887145228683949\n",
      "Episode 22. Training Carried out at step = 250. Actorloss = 0.02787620574235916, Critic_loss = 0.007993629202246666\n",
      "Episode 22. Training Carried out at step = 260. Actorloss = 0.02820800431072712, Critic_loss = 0.004970823414623737\n",
      "Episode 22. Training Carried out at step = 270. Actorloss = 0.02851923741400242, Critic_loss = 0.0031522796489298344\n",
      "Episode 22. Training Carried out at step = 280. Actorloss = 0.02881024219095707, Critic_loss = 0.0025394735857844353\n",
      "Episode 22. Training Carried out at step = 290. Actorloss = 0.029083725064992905, Critic_loss = 0.0029308097437024117\n",
      "Episode 22. Training Carried out at step = 300. Actorloss = 0.02934243716299534, Critic_loss = 0.0039570326916873455\n",
      "Episode 22. Training Carried out at step = 310. Actorloss = 0.029589615762233734, Critic_loss = 0.005288160406053066\n",
      "Episode 22. Training Carried out at step = 320. Actorloss = 0.029828939586877823, Critic_loss = 0.006332450080662966\n",
      "Episode 22. Training Carried out at step = 330. Actorloss = 0.030064161866903305, Critic_loss = 0.007058336399495602\n",
      "Episode 22. Training Carried out at step = 340. Actorloss = 0.030298909172415733, Critic_loss = 0.007236212491989136\n",
      "Episode 22. Training Carried out at step = 350. Actorloss = 0.03053642064332962, Critic_loss = 0.006891725584864616\n",
      "Episode 22. Training Carried out at step = 360. Actorloss = 0.030778713524341583, Critic_loss = 0.006157412193715572\n",
      "Episode 22. Training Carried out at step = 370. Actorloss = 0.03102826327085495, Critic_loss = 0.005196319427341223\n",
      "Episode 22. Training Carried out at step = 380. Actorloss = 0.03128715977072716, Critic_loss = 0.0042626685462892056\n",
      "Episode 22. Training Carried out at step = 390. Actorloss = 0.03155684471130371, Critic_loss = 0.003436693921685219\n",
      "Episode 22. Training Carried out at step = 400. Actorloss = 0.03183682635426521, Critic_loss = 0.0028695485088974237\n",
      "Episode 22. Training Carried out at step = 410. Actorloss = 0.032127540558576584, Critic_loss = 0.0025647839065641165\n",
      "Episode 22. Training Carried out at step = 420. Actorloss = 0.0324285589158535, Critic_loss = 0.002510879887267947\n",
      "Episode 22. Training Carried out at step = 430. Actorloss = 0.0327388234436512, Critic_loss = 0.002640113001689315\n",
      "Episode 22. Training Carried out at step = 440. Actorloss = 0.03304029628634453, Critic_loss = 0.0028690588660538197\n",
      "Episode 22. Training Carried out at step = 450. Actorloss = 0.03334576264023781, Critic_loss = 0.0031299772672355175\n",
      "Episode 22. Training Carried out at step = 460. Actorloss = 0.03365562483668327, Critic_loss = 0.003331945277750492\n",
      "Episode 22. Training Carried out at step = 470. Actorloss = 0.03396816551685333, Critic_loss = 0.0034506218507885933\n",
      "Episode 22. Training Carried out at step = 480. Actorloss = 0.03428079932928085, Critic_loss = 0.003472430631518364\n",
      "Episode 22. Training Carried out at step = 490. Actorloss = 0.034592896699905396, Critic_loss = 0.0033801416866481304\n",
      "Episode 22. Training Carried out at step = 500. Actorloss = 0.034903351217508316, Critic_loss = 0.0032257759012281895\n",
      "Episode 22. Training Carried out at step = 510. Actorloss = 0.03521110862493515, Critic_loss = 0.0030233673751354218\n",
      "Episode 22. Training Carried out at step = 520. Actorloss = 0.035515423864126205, Critic_loss = 0.002826379844918847\n",
      "Episode 22. Training Carried out at step = 530. Actorloss = 0.035815805196762085, Critic_loss = 0.002654594834893942\n",
      "Episode 22. Training Carried out at step = 540. Actorloss = 0.03611205890774727, Critic_loss = 0.0025376344565302134\n",
      "Episode 22. Training Carried out at step = 550. Actorloss = 0.03640426695346832, Critic_loss = 0.0024731329176574945\n",
      "Episode 22. Training Carried out at step = 560. Actorloss = 0.03669267147779465, Critic_loss = 0.002463421318680048\n",
      "Episode 22. Training Carried out at step = 570. Actorloss = 0.03697776421904564, Critic_loss = 0.0024949079379439354\n",
      "Episode 22. Training Carried out at step = 580. Actorloss = 0.03726005554199219, Critic_loss = 0.002550621749833226\n",
      "Episode 22. Training Carried out at step = 590. Actorloss = 0.03754017502069473, Critic_loss = 0.0026126003358513117\n",
      "Episode 22. Training Carried out at step = 600. Actorloss = 0.03781905770301819, Critic_loss = 0.002660031896084547\n",
      "Episode 22. Training Carried out at step = 610. Actorloss = 0.038097504526376724, Critic_loss = 0.002692761132493615\n",
      "Episode 22. Training Carried out at step = 620. Actorloss = 0.03837142512202263, Critic_loss = 0.0026944971177726984\n",
      "Episode 22. Training Carried out at step = 630. Actorloss = 0.038645319640636444, Critic_loss = 0.0026787330862134695\n",
      "Episode 22. Training Carried out at step = 640. Actorloss = 0.038919493556022644, Critic_loss = 0.002634099218994379\n",
      "Episode 22. Training Carried out at step = 650. Actorloss = 0.03919490799307823, Critic_loss = 0.002586928429082036\n",
      "Episode 22. Training Carried out at step = 660. Actorloss = 0.03947294130921364, Critic_loss = 0.002534199273213744\n",
      "Episode 22. Training Carried out at step = 670. Actorloss = 0.03975389897823334, Critic_loss = 0.0024869446642696857\n",
      "Episode 22. Training Carried out at step = 680. Actorloss = 0.04003788158297539, Critic_loss = 0.002450905041769147\n",
      "Episode 22. Training Carried out at step = 690. Actorloss = 0.0403243787586689, Critic_loss = 0.0024269523564726114\n",
      "Episode 22. Training Carried out at step = 700. Actorloss = 0.040613092482089996, Critic_loss = 0.002416131552308798\n",
      "Episode 22. Training Carried out at step = 710. Actorloss = 0.040904451161623, Critic_loss = 0.0024156784638762474\n",
      "Episode 22. Training Carried out at step = 720. Actorloss = 0.04119812697172165, Critic_loss = 0.0024187189992517233\n",
      "Episode 22. Training Carried out at step = 730. Actorloss = 0.04149379953742027, Critic_loss = 0.0024300802033394575\n",
      "Episode 22. Training Carried out at step = 740. Actorloss = 0.04179082438349724, Critic_loss = 0.002437492134049535\n",
      "Episode 22. Training Carried out at step = 750. Actorloss = 0.04207446053624153, Critic_loss = 0.0024416521191596985\n",
      "Episode 22. Training Carried out at step = 760. Actorloss = 0.04235508665442467, Critic_loss = 0.0024407135788351297\n",
      "Episode 22. Training Carried out at step = 770. Actorloss = 0.04263605549931526, Critic_loss = 0.002433974761515856\n",
      "Episode 22. Training Carried out at step = 780. Actorloss = 0.042917024344205856, Critic_loss = 0.0024244883097708225\n",
      "Episode 22. Training Carried out at step = 790. Actorloss = 0.043197374790906906, Critic_loss = 0.0024143708869814873\n",
      "Episode 22. Training Carried out at step = 800. Actorloss = 0.04347663372755051, Critic_loss = 0.00240209698677063\n",
      "Episode 22. Training Carried out at step = 810. Actorloss = 0.04375533387064934, Critic_loss = 0.0023904868867248297\n",
      "Episode 22. Training Carried out at step = 820. Actorloss = 0.044033411890268326, Critic_loss = 0.0023820146452635527\n",
      "Episode 22. Training Carried out at step = 830. Actorloss = 0.04431084915995598, Critic_loss = 0.002376879332587123\n",
      "Episode 22. Training Carried out at step = 840. Actorloss = 0.04458772763609886, Critic_loss = 0.0023741924669593573\n",
      "Episode 22. Training Carried out at step = 850. Actorloss = 0.044864118099212646, Critic_loss = 0.0023739030584692955\n",
      "Episode 22. Training Carried out at step = 860. Actorloss = 0.04514019191265106, Critic_loss = 0.00237485533580184\n",
      "Episode 22. Training Carried out at step = 870. Actorloss = 0.04541610926389694, Critic_loss = 0.002375876996666193\n",
      "Episode 22. Training Carried out at step = 880. Actorloss = 0.045692045241594315, Critic_loss = 0.0023766965605318546\n",
      "Episode 22. Training Carried out at step = 890. Actorloss = 0.04596634954214096, Critic_loss = 0.002376059303060174\n",
      "Episode 22. Training Carried out at step = 900. Actorloss = 0.04623773694038391, Critic_loss = 0.002373599214479327\n",
      "Episode 22. Training Carried out at step = 910. Actorloss = 0.04650888592004776, Critic_loss = 0.0023733251728117466\n",
      "Episode 22. Training Carried out at step = 920. Actorloss = 0.046780526638031006, Critic_loss = 0.0023647143971174955\n",
      "Episode 22. Training Carried out at step = 930. Actorloss = 0.0470527708530426, Critic_loss = 0.0023583995644003153\n",
      "Episode 22. Training Carried out at step = 940. Actorloss = 0.047325924038887024, Critic_loss = 0.0023528188467025757\n",
      "Episode 22. Training Carried out at step = 950. Actorloss = 0.04760008305311203, Critic_loss = 0.0023471475578844547\n",
      "Episode 22. Training Carried out at step = 960. Actorloss = 0.047875288873910904, Critic_loss = 0.0023418404161930084\n",
      "Episode 22. Training Carried out at step = 970. Actorloss = 0.04815151169896126, Critic_loss = 0.0023372441064566374\n",
      "Episode 22. Training Carried out at step = 980. Actorloss = 0.0484287291765213, Critic_loss = 0.0023332699202001095\n",
      "Episode 22. Training Carried out at step = 990. Actorloss = 0.04870687797665596, Critic_loss = 0.0023301260080188513\n",
      "Episode 22. Training Carried out at step = 1000. Actorloss = 0.048985883593559265, Critic_loss = 0.0023273397237062454\n",
      "Episode 23. Training Carried out at step = 10. Actorloss = 0.0030376643408089876, Critic_loss = 0.1525079905986786\n",
      "Episode 23. Training Carried out at step = 20. Actorloss = -0.00953010842204094, Critic_loss = 3.04776930809021\n",
      "Episode 23. Training Carried out at step = 30. Actorloss = 0.021803906187415123, Critic_loss = 9.481925010681152\n",
      "Episode 23. Training Carried out at step = 40. Actorloss = 0.0026291769463568926, Critic_loss = 15.606316566467285\n",
      "Episode 23. Training Carried out at step = 50. Actorloss = 0.02350519970059395, Critic_loss = 2.6724772453308105\n",
      "Episode 23. Training Carried out at step = 60. Actorloss = 0.10720350593328476, Critic_loss = 0.024402201175689697\n",
      "Episode 23. Training Carried out at step = 70. Actorloss = 0.10672417283058167, Critic_loss = 0.025577453896403313\n",
      "Episode 23. Training Carried out at step = 80. Actorloss = 0.10613439977169037, Critic_loss = 0.023988865315914154\n",
      "Episode 23. Training Carried out at step = 90. Actorloss = 0.10542543232440948, Critic_loss = 0.020237544551491737\n",
      "Episode 23. Training Carried out at step = 100. Actorloss = 0.10377685725688934, Critic_loss = 0.015159696340560913\n",
      "Episode 23. Training Carried out at step = 110. Actorloss = 0.1013389378786087, Critic_loss = 0.010005207732319832\n",
      "Episode 23. Training Carried out at step = 120. Actorloss = 0.09862066805362701, Critic_loss = 0.005819747224450111\n",
      "Episode 23. Training Carried out at step = 130. Actorloss = 0.09589843451976776, Critic_loss = 0.003284396603703499\n",
      "Episode 23. Training Carried out at step = 140. Actorloss = 0.09319580346345901, Critic_loss = 0.0025003422051668167\n",
      "Episode 23. Training Carried out at step = 150. Actorloss = 0.09053006768226624, Critic_loss = 0.003186785150319338\n",
      "Episode 23. Training Carried out at step = 160. Actorloss = 0.08790361136198044, Critic_loss = 0.004822997376322746\n",
      "Episode 23. Training Carried out at step = 170. Actorloss = 0.08525658398866653, Critic_loss = 0.006777776405215263\n",
      "Episode 23. Training Carried out at step = 180. Actorloss = 0.0826048031449318, Critic_loss = 0.008473280817270279\n",
      "Episode 23. Training Carried out at step = 190. Actorloss = 0.08140502870082855, Critic_loss = 0.009289402514696121\n",
      "Episode 23. Training Carried out at step = 200. Actorloss = 0.08118234574794769, Critic_loss = 0.009114555083215237\n",
      "Episode 23. Training Carried out at step = 210. Actorloss = 0.07830440253019333, Critic_loss = 0.008604474365711212\n",
      "Episode 23. Training Carried out at step = 220. Actorloss = 0.05478968471288681, Critic_loss = 0.01003074087202549\n",
      "Episode 23. Training Carried out at step = 230. Actorloss = 0.05533789470791817, Critic_loss = 0.007426103111356497\n",
      "Episode 23. Training Carried out at step = 240. Actorloss = 0.05563293397426605, Critic_loss = 0.005058407783508301\n",
      "Episode 23. Training Carried out at step = 250. Actorloss = 0.055910881608724594, Critic_loss = 0.0033365576528012753\n",
      "Episode 23. Training Carried out at step = 260. Actorloss = 0.056173283606767654, Critic_loss = 0.002569491509348154\n",
      "Episode 23. Training Carried out at step = 270. Actorloss = 0.05642138794064522, Critic_loss = 0.002723448211327195\n",
      "Episode 23. Training Carried out at step = 280. Actorloss = 0.05665716156363487, Critic_loss = 0.003495516488328576\n",
      "Episode 23. Training Carried out at step = 290. Actorloss = 0.056883554905653, Critic_loss = 0.004472721833735704\n",
      "Episode 23. Training Carried out at step = 300. Actorloss = 0.05710367485880852, Critic_loss = 0.005247302353382111\n",
      "Episode 23. Training Carried out at step = 310. Actorloss = 0.05732036754488945, Critic_loss = 0.0055747367441654205\n",
      "Episode 23. Training Carried out at step = 320. Actorloss = 0.05753760784864426, Critic_loss = 0.005375775974243879\n",
      "Episode 23. Training Carried out at step = 330. Actorloss = 0.05775842443108559, Critic_loss = 0.004796824883669615\n",
      "Episode 23. Training Carried out at step = 340. Actorloss = 0.057985007762908936, Critic_loss = 0.0040385606698691845\n",
      "Episode 23. Training Carried out at step = 350. Actorloss = 0.05821087583899498, Critic_loss = 0.0033197638113051653\n",
      "Episode 23. Training Carried out at step = 360. Actorloss = 0.05843440815806389, Critic_loss = 0.0027977910358458757\n",
      "Episode 23. Training Carried out at step = 370. Actorloss = 0.058663513511419296, Critic_loss = 0.0025364079046994448\n",
      "Episode 23. Training Carried out at step = 380. Actorloss = 0.05890033766627312, Critic_loss = 0.002522245980799198\n",
      "Episode 23. Training Carried out at step = 390. Actorloss = 0.059143904596567154, Critic_loss = 0.0026792900171130896\n",
      "Episode 23. Training Carried out at step = 400. Actorloss = 0.05939250811934471, Critic_loss = 0.002911000046879053\n",
      "Episode 23. Training Carried out at step = 410. Actorloss = 0.05964392423629761, Critic_loss = 0.0031206996645778418\n",
      "Episode 23. Training Carried out at step = 420. Actorloss = 0.05988562852144241, Critic_loss = 0.003245628671720624\n",
      "Episode 23. Training Carried out at step = 430. Actorloss = 0.06012708693742752, Critic_loss = 0.003261236008256674\n",
      "Episode 23. Training Carried out at step = 440. Actorloss = 0.06036584451794624, Critic_loss = 0.0031744621228426695\n",
      "Episode 23. Training Carried out at step = 450. Actorloss = 0.06059299036860466, Critic_loss = 0.003019754309207201\n",
      "Episode 23. Training Carried out at step = 460. Actorloss = 0.060814641416072845, Critic_loss = 0.002843234222382307\n",
      "Episode 23. Training Carried out at step = 470. Actorloss = 0.0610298216342926, Critic_loss = 0.0026782206259667873\n",
      "Episode 23. Training Carried out at step = 480. Actorloss = 0.06123720109462738, Critic_loss = 0.0025548585690557957\n",
      "Episode 23. Training Carried out at step = 490. Actorloss = 0.06144052743911743, Critic_loss = 0.002482392592355609\n",
      "Episode 23. Training Carried out at step = 500. Actorloss = 0.061640020459890366, Critic_loss = 0.0024645235389471054\n",
      "Episode 23. Training Carried out at step = 510. Actorloss = 0.06183489412069321, Critic_loss = 0.0024890918284654617\n",
      "Episode 23. Training Carried out at step = 520. Actorloss = 0.062026407569646835, Critic_loss = 0.002537031192332506\n",
      "Episode 23. Training Carried out at step = 530. Actorloss = 0.06221587955951691, Critic_loss = 0.0025940006598830223\n",
      "Episode 23. Training Carried out at step = 540. Actorloss = 0.06240411102771759, Critic_loss = 0.0026404969394207\n",
      "Episode 23. Training Carried out at step = 550. Actorloss = 0.0625908151268959, Critic_loss = 0.002665645210072398\n",
      "Episode 23. Training Carried out at step = 560. Actorloss = 0.06277770549058914, Critic_loss = 0.002661103615537286\n",
      "Episode 23. Training Carried out at step = 570. Actorloss = 0.0629652664065361, Critic_loss = 0.0026355781592428684\n",
      "Episode 23. Training Carried out at step = 580. Actorloss = 0.06315334141254425, Critic_loss = 0.0025921151973307133\n",
      "Episode 23. Training Carried out at step = 590. Actorloss = 0.06334298104047775, Critic_loss = 0.0025465732906013727\n",
      "Episode 23. Training Carried out at step = 600. Actorloss = 0.06353411823511124, Critic_loss = 0.00249806372448802\n",
      "Episode 23. Training Carried out at step = 610. Actorloss = 0.06372259557247162, Critic_loss = 0.002460629213601351\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-53fe032dce0a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"episode_count set to \"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepisode_count\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m         \u001b[0mworker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwork\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcoord\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msaver\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglobal_summary\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Start Training / Testing operation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[0;31m# if not TRAINING:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-19-132ba1192f80>\u001b[0m in \u001b[0;36mwork\u001b[0;34m(self, sess, coord, saver, summary)\u001b[0m\n\u001b[1;32m     91\u001b[0m                     \u001b[0;31m# Obtain the action as output by our actor network based on current state\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m                     \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_network\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobservation_input_placeholder\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m                     \u001b[0ma\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglobal_network\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mactor_net_output\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mTRAINING\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/me5418-p1/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    885\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 887\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    888\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    889\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/me5418-p1/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1108\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1109\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1110\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1111\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1112\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/me5418-p1/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1284\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1285\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1286\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1287\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1288\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/me5418-p1/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1290\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1291\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1292\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1293\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1294\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/me5418-p1/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1275\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1276\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1277\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1278\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1279\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/me5418-p1/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1365\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1366\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1367\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1368\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1369\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "print(\"Hello World!\")\n",
    "if not os.path.exists(model_path):\n",
    "    os.makedirs(model_path)\n",
    "if not os.path.exists(gifs_path):\n",
    "    os.makedirs(gifs_path)\n",
    "if not os.path.exists(train_path):\n",
    "    os.makedirs(train_path)\n",
    "if not os.path.exists(episode_buffer_path):\n",
    "    os.makedirs(episode_buffer_path)\n",
    "\n",
    "config = tf.ConfigProto(allow_soft_placement = True) #  to allow the placement of an operation on a device other than the one explicitly requested\n",
    "\n",
    "with tf.device(\"/cpu:0\"): # for now just use cpu to run. Might consider changing to gpu in the future to speed up training process.\n",
    "     \n",
    "    master_network = DDPGNet(observe_size=observe_size, action_size=action_size, TRAINING=TRAINING, gamma=GAMMA, tau=TAU, actor_lr=ACTOR_LR, actor_l2_reg=ACTOR_L2_REG, actor_grad_clip=ACTOR_GRAD_CLIP, critic_lr=CRITIC_LR, critic_l2_reg=CRITIC_L2_REG, critic_grad_clip=CRITIC_GRAD_CLIP, batch_size=BATCH_SIZE)\n",
    "\n",
    "    gameEnv = AliengoEnv(render = True, Policy = Policy) # For us to visualize how each episode runs\n",
    "    gameEnv.reset(reload_urdf=True) # To import the model in\n",
    "    noise = OUNoise(MU, THETA, SIGMA, DECAY_R) # initializing OU Noise\n",
    "    noise.reset() # reset the OU noise\n",
    "    worker = Worker(gameEnv, master_network, noise) # Initialize our Worker Agent\n",
    "\n",
    "    global_summary = tf.summary.FileWriter(train_path) # Initializing our tf summary -> for tensorboard visualisation\n",
    "    saver = tf.train.Saver(max_to_keep = 2) # Initializing our tf saver -> for saving -> for saving model checkpoint\n",
    "\n",
    "    with tf.Session(config=config) as sess:\n",
    "        sess.run(tf.global_variables_initializer())\n",
    "        coord = tf.train.Coordinator()\n",
    "\n",
    "        if load_model == True: # To load the model back from our tf checkpoint files\n",
    "            print(\"Loading Model...\")\n",
    "            if not TRAINING:\n",
    "                with open(model_path+'/checkpoint', 'w') as file:\n",
    "                    file.write('model_checkpoint_path: \"model-{}.ckpt\"'.format(MODEL_NUMBER))\n",
    "                    file.close()\n",
    "            ckpt = tf.train.get_checkpoint_state(model_path)\n",
    "            p=ckpt.model_checkpoint_path\n",
    "            p=p[p.find('-')+1:]\n",
    "            p=p[:p.find('.')]\n",
    "            if TRAINING:\n",
    "                episode_count = int(p)\n",
    "            else:\n",
    "                episode_count = 0\n",
    "            saver.restore(sess,ckpt.model_checkpoint_path)\n",
    "            print(\"episode_count set to \", episode_count)\n",
    "         \n",
    "        worker.work(sess, coord, saver, global_summary) # Start Training / Testing operation\n",
    "            \n",
    "# if not TRAINING:\n",
    "#     print([np.mean(episode_lengths), np.sqrt(np.var(episode_lengths)), np.mean(np.asarray(np.asarray(episode_lengths) < MAX_EPISODE_LENGTH, dtype=float))])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Disconnect Cell\n",
    "If we manually end the training operation by stopping the cell above (Keyboard Interuption), pybullet window would still be running. \n",
    "\n",
    "Run the cell below to properly terminate the pybullet window"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "gameEnv.disconnect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "me5418-p1",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}