frozenLake.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "0.15.3\n"
    }
   ],
   "source": [
    "import gym\n",
    "print(gym.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import time\n",
    "import random\n",
    "from IPython.display import clear_output\n",
    "from tqdm import tqdm_notebook as tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "env= gym.make('FrozenLake-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16, 4)"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state_space_size= env.observation_space.n\n",
    "action_space_size= env.action_space.n\n",
    "q_table= np.zeros((state_space_size, action_space_size))\n",
    "q_table.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_episodes= 1000000\n",
    "max_steps_per_episode= 100\n",
    "\n",
    "learning_rate= 0.01\n",
    "discount_rate= 0.99\n",
    "\n",
    "exploration_rate= 1\n",
    "max_exploration_rate= 1\n",
    "min_exploration_rate= 0.01\n",
    "exploration_decay_rate= 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7b26db26daa47e0aff57df8669f4ce9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1000000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Q-learning implementation:\n",
    "rewards_all_episode= []\n",
    "\n",
    "for episode in tqdm(range(num_episodes)):\n",
    "    # initialize a new episode\n",
    "    state= env.reset()\n",
    "    done= False\n",
    "    reward_current_episode= 0\n",
    "\n",
    "    for step in range(max_steps_per_episode):\n",
    "        exploration_rate_threshold= random.uniform(0,1)\n",
    "        # if random choosen task by the agent is exploration\n",
    "        if exploration_rate_threshold > exploration_rate:\n",
    "            action= np.argmax(q_table[state, :])\n",
    "        # if randomly choosen task by the agent is exploitation\n",
    "        else:\n",
    "            action= env.action_space.sample()\n",
    "        \n",
    "        \n",
    "        # taking action\n",
    "        new_state, reward, done, info= env.step(action)\n",
    "        # update the q table based on the reward\n",
    "        q_table[state, action]= ((1 - learning_rate) * q_table[state, action]) +\\\n",
    "        (learning_rate * (reward + discount_rate * np.max(q_table[new_state, :])))\n",
    "        \n",
    "        # transition to the next state\n",
    "        state= new_state\n",
    "        reward_current_episode += reward\n",
    "        \n",
    "        # exit episode\n",
    "        if done == True:\n",
    "            break\n",
    "            \n",
    "    # exploration rate exponential decay\n",
    "    exploration_rate= min_exploration_rate +\\\n",
    "    (max_exploration_rate - min_exploration_rate) * np.exp(-exploration_decay_rate * episode)\n",
    "    \n",
    "    rewards_all_episode.append(reward_current_episode)\n",
    "    \n",
    "#     # calculate and print average reward per thousand\n",
    "#     if episode % 1000 == 0 and episode != 0:\n",
    "#         print('Reward on %d th episode : %.2f, final reward score: %.2f'\\\n",
    "#               % (episode, reward_current_episode, sum(rewards_all_episode)/episode ))\n",
    "\n",
    "        \n",
    "        \n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "********Average reward per thousand episodes********\n",
      "\n",
      "1000 : 0.025\n",
      "2000 : 0.065\n",
      "3000 : 0.210\n",
      "4000 : 0.267\n",
      "5000 : 0.282\n",
      "6000 : 0.266\n",
      "7000 : 0.288\n",
      "8000 : 0.279\n",
      "9000 : 0.280\n",
      "10000 : 0.291\n",
      "11000 : 0.288\n",
      "12000 : 0.320\n",
      "13000 : 0.318\n",
      "14000 : 0.301\n",
      "15000 : 0.302\n",
      "16000 : 0.560\n",
      "17000 : 0.538\n",
      "18000 : 0.545\n",
      "19000 : 0.544\n",
      "20000 : 0.537\n",
      "21000 : 0.511\n",
      "22000 : 0.529\n",
      "23000 : 0.524\n",
      "24000 : 0.555\n",
      "25000 : 0.540\n",
      "26000 : 0.531\n",
      "27000 : 0.557\n",
      "28000 : 0.540\n",
      "29000 : 0.538\n",
      "30000 : 0.551\n",
      "31000 : 0.536\n",
      "32000 : 0.526\n",
      "33000 : 0.549\n",
      "34000 : 0.567\n",
      "35000 : 0.545\n",
      "36000 : 0.546\n",
      "37000 : 0.548\n",
      "38000 : 0.556\n",
      "39000 : 0.535\n",
      "40000 : 0.529\n",
      "41000 : 0.527\n",
      "42000 : 0.558\n",
      "43000 : 0.551\n",
      "44000 : 0.516\n",
      "45000 : 0.549\n",
      "46000 : 0.551\n",
      "47000 : 0.550\n",
      "48000 : 0.552\n",
      "49000 : 0.552\n",
      "50000 : 0.539\n",
      "51000 : 0.540\n",
      "52000 : 0.579\n",
      "53000 : 0.566\n",
      "54000 : 0.590\n",
      "55000 : 0.602\n",
      "56000 : 0.595\n",
      "57000 : 0.576\n",
      "58000 : 0.573\n",
      "59000 : 0.601\n",
      "60000 : 0.599\n",
      "61000 : 0.628\n",
      "62000 : 0.598\n",
      "63000 : 0.591\n",
      "64000 : 0.620\n",
      "65000 : 0.610\n",
      "66000 : 0.674\n",
      "67000 : 0.674\n",
      "68000 : 0.690\n",
      "69000 : 0.687\n",
      "70000 : 0.681\n",
      "71000 : 0.679\n",
      "72000 : 0.693\n",
      "73000 : 0.695\n",
      "74000 : 0.661\n",
      "75000 : 0.682\n",
      "76000 : 0.672\n",
      "77000 : 0.686\n",
      "78000 : 0.700\n",
      "79000 : 0.678\n",
      "80000 : 0.677\n",
      "81000 : 0.703\n",
      "82000 : 0.686\n",
      "83000 : 0.691\n",
      "84000 : 0.681\n",
      "85000 : 0.674\n",
      "86000 : 0.671\n",
      "87000 : 0.662\n",
      "88000 : 0.699\n",
      "89000 : 0.697\n",
      "90000 : 0.685\n",
      "91000 : 0.668\n",
      "92000 : 0.696\n",
      "93000 : 0.702\n",
      "94000 : 0.678\n",
      "95000 : 0.715\n",
      "96000 : 0.684\n",
      "97000 : 0.689\n",
      "98000 : 0.690\n",
      "99000 : 0.704\n",
      "100000 : 0.694\n",
      "101000 : 0.706\n",
      "102000 : 0.659\n",
      "103000 : 0.689\n",
      "104000 : 0.665\n",
      "105000 : 0.715\n",
      "106000 : 0.650\n",
      "107000 : 0.690\n",
      "108000 : 0.685\n",
      "109000 : 0.694\n",
      "110000 : 0.676\n",
      "111000 : 0.660\n",
      "112000 : 0.684\n",
      "113000 : 0.684\n",
      "114000 : 0.669\n",
      "115000 : 0.696\n",
      "116000 : 0.695\n",
      "117000 : 0.682\n",
      "118000 : 0.697\n",
      "119000 : 0.687\n",
      "120000 : 0.655\n",
      "121000 : 0.670\n",
      "122000 : 0.704\n",
      "123000 : 0.688\n",
      "124000 : 0.675\n",
      "125000 : 0.681\n",
      "126000 : 0.700\n",
      "127000 : 0.683\n",
      "128000 : 0.695\n",
      "129000 : 0.687\n",
      "130000 : 0.684\n",
      "131000 : 0.679\n",
      "132000 : 0.711\n",
      "133000 : 0.683\n",
      "134000 : 0.672\n",
      "135000 : 0.681\n",
      "136000 : 0.697\n",
      "137000 : 0.690\n",
      "138000 : 0.713\n",
      "139000 : 0.662\n",
      "140000 : 0.675\n",
      "141000 : 0.682\n",
      "142000 : 0.683\n",
      "143000 : 0.700\n",
      "144000 : 0.689\n",
      "145000 : 0.669\n",
      "146000 : 0.687\n",
      "147000 : 0.692\n",
      "148000 : 0.703\n",
      "149000 : 0.655\n",
      "150000 : 0.692\n",
      "151000 : 0.679\n",
      "152000 : 0.687\n",
      "153000 : 0.668\n",
      "154000 : 0.692\n",
      "155000 : 0.678\n",
      "156000 : 0.687\n",
      "157000 : 0.681\n",
      "158000 : 0.674\n",
      "159000 : 0.713\n",
      "160000 : 0.689\n",
      "161000 : 0.689\n",
      "162000 : 0.670\n",
      "163000 : 0.722\n",
      "164000 : 0.692\n",
      "165000 : 0.661\n",
      "166000 : 0.671\n",
      "167000 : 0.694\n",
      "168000 : 0.697\n",
      "169000 : 0.706\n",
      "170000 : 0.657\n",
      "171000 : 0.697\n",
      "172000 : 0.681\n",
      "173000 : 0.703\n",
      "174000 : 0.686\n",
      "175000 : 0.705\n",
      "176000 : 0.708\n",
      "177000 : 0.678\n",
      "178000 : 0.695\n",
      "179000 : 0.689\n",
      "180000 : 0.657\n",
      "181000 : 0.681\n",
      "182000 : 0.661\n",
      "183000 : 0.672\n",
      "184000 : 0.692\n",
      "185000 : 0.672\n",
      "186000 : 0.678\n",
      "187000 : 0.679\n",
      "188000 : 0.699\n",
      "189000 : 0.665\n",
      "190000 : 0.685\n",
      "191000 : 0.693\n",
      "192000 : 0.686\n",
      "193000 : 0.710\n",
      "194000 : 0.711\n",
      "195000 : 0.675\n",
      "196000 : 0.686\n",
      "197000 : 0.687\n",
      "198000 : 0.682\n",
      "199000 : 0.699\n",
      "200000 : 0.706\n",
      "201000 : 0.673\n",
      "202000 : 0.699\n",
      "203000 : 0.684\n",
      "204000 : 0.664\n",
      "205000 : 0.686\n",
      "206000 : 0.690\n",
      "207000 : 0.677\n",
      "208000 : 0.675\n",
      "209000 : 0.691\n",
      "210000 : 0.704\n",
      "211000 : 0.700\n",
      "212000 : 0.667\n",
      "213000 : 0.690\n",
      "214000 : 0.706\n",
      "215000 : 0.667\n",
      "216000 : 0.678\n",
      "217000 : 0.684\n",
      "218000 : 0.668\n",
      "219000 : 0.671\n",
      "220000 : 0.682\n",
      "221000 : 0.670\n",
      "222000 : 0.699\n",
      "223000 : 0.664\n",
      "224000 : 0.691\n",
      "225000 : 0.690\n",
      "226000 : 0.701\n",
      "227000 : 0.679\n",
      "228000 : 0.690\n",
      "229000 : 0.700\n",
      "230000 : 0.672\n",
      "231000 : 0.711\n",
      "232000 : 0.707\n",
      "233000 : 0.679\n",
      "234000 : 0.701\n",
      "235000 : 0.682\n",
      "236000 : 0.694\n",
      "237000 : 0.703\n",
      "238000 : 0.705\n",
      "239000 : 0.692\n",
      "240000 : 0.685\n",
      "241000 : 0.687\n",
      "242000 : 0.711\n",
      "243000 : 0.693\n",
      "244000 : 0.698\n",
      "245000 : 0.679\n",
      "246000 : 0.684\n",
      "247000 : 0.719\n",
      "248000 : 0.693\n",
      "249000 : 0.701\n",
      "250000 : 0.687\n",
      "251000 : 0.674\n",
      "252000 : 0.706\n",
      "253000 : 0.659\n",
      "254000 : 0.701\n",
      "255000 : 0.687\n",
      "256000 : 0.687\n",
      "257000 : 0.693\n",
      "258000 : 0.688\n",
      "259000 : 0.675\n",
      "260000 : 0.674\n",
      "261000 : 0.666\n",
      "262000 : 0.683\n",
      "263000 : 0.703\n",
      "264000 : 0.688\n",
      "265000 : 0.663\n",
      "266000 : 0.682\n",
      "267000 : 0.691\n",
      "268000 : 0.688\n",
      "269000 : 0.673\n",
      "270000 : 0.685\n",
      "271000 : 0.692\n",
      "272000 : 0.675\n",
      "273000 : 0.677\n",
      "274000 : 0.671\n",
      "275000 : 0.667\n",
      "276000 : 0.676\n",
      "277000 : 0.693\n",
      "278000 : 0.675\n",
      "279000 : 0.660\n",
      "280000 : 0.684\n",
      "281000 : 0.684\n",
      "282000 : 0.676\n",
      "283000 : 0.697\n",
      "284000 : 0.674\n",
      "285000 : 0.687\n",
      "286000 : 0.704\n",
      "287000 : 0.665\n",
      "288000 : 0.698\n",
      "289000 : 0.708\n",
      "290000 : 0.680\n",
      "291000 : 0.674\n",
      "292000 : 0.664\n",
      "293000 : 0.678\n",
      "294000 : 0.698\n",
      "295000 : 0.666\n",
      "296000 : 0.704\n",
      "297000 : 0.688\n",
      "298000 : 0.672\n",
      "299000 : 0.671\n",
      "300000 : 0.699\n",
      "301000 : 0.689\n",
      "302000 : 0.686\n",
      "303000 : 0.686\n",
      "304000 : 0.697\n",
      "305000 : 0.695\n",
      "306000 : 0.693\n",
      "307000 : 0.692\n",
      "308000 : 0.712\n",
      "309000 : 0.693\n",
      "310000 : 0.691\n",
      "311000 : 0.680\n",
      "312000 : 0.703\n",
      "313000 : 0.671\n",
      "314000 : 0.707\n",
      "315000 : 0.671\n",
      "316000 : 0.693\n",
      "317000 : 0.650\n",
      "318000 : 0.689\n",
      "319000 : 0.694\n",
      "320000 : 0.699\n",
      "321000 : 0.678\n",
      "322000 : 0.674\n",
      "323000 : 0.694\n",
      "324000 : 0.680\n",
      "325000 : 0.667\n",
      "326000 : 0.680\n",
      "327000 : 0.700\n",
      "328000 : 0.670\n",
      "329000 : 0.693\n",
      "330000 : 0.697\n",
      "331000 : 0.722\n",
      "332000 : 0.693\n",
      "333000 : 0.673\n",
      "334000 : 0.685\n",
      "335000 : 0.680\n",
      "336000 : 0.678\n",
      "337000 : 0.667\n",
      "338000 : 0.677\n",
      "339000 : 0.692\n",
      "340000 : 0.678\n",
      "341000 : 0.680\n",
      "342000 : 0.682\n",
      "343000 : 0.692\n",
      "344000 : 0.724\n",
      "345000 : 0.661\n",
      "346000 : 0.649\n",
      "347000 : 0.683\n",
      "348000 : 0.684\n",
      "349000 : 0.681\n",
      "350000 : 0.689\n",
      "351000 : 0.671\n",
      "352000 : 0.706\n",
      "353000 : 0.673\n",
      "354000 : 0.684\n",
      "355000 : 0.683\n",
      "356000 : 0.692\n",
      "357000 : 0.716\n",
      "358000 : 0.689\n",
      "359000 : 0.671\n",
      "360000 : 0.676\n",
      "361000 : 0.669\n",
      "362000 : 0.683\n",
      "363000 : 0.669\n",
      "364000 : 0.689\n",
      "365000 : 0.692\n",
      "366000 : 0.655\n",
      "367000 : 0.683\n",
      "368000 : 0.696\n",
      "369000 : 0.655\n",
      "370000 : 0.704\n",
      "371000 : 0.702\n",
      "372000 : 0.694\n",
      "373000 : 0.680\n",
      "374000 : 0.684\n",
      "375000 : 0.683\n",
      "376000 : 0.690\n",
      "377000 : 0.704\n",
      "378000 : 0.658\n",
      "379000 : 0.682\n",
      "380000 : 0.691\n",
      "381000 : 0.681\n",
      "382000 : 0.689\n",
      "383000 : 0.709\n",
      "384000 : 0.669\n",
      "385000 : 0.680\n",
      "386000 : 0.698\n",
      "387000 : 0.704\n",
      "388000 : 0.705\n",
      "389000 : 0.687\n",
      "390000 : 0.702\n",
      "391000 : 0.699\n",
      "392000 : 0.667\n",
      "393000 : 0.678\n",
      "394000 : 0.672\n",
      "395000 : 0.684\n",
      "396000 : 0.661\n",
      "397000 : 0.675\n",
      "398000 : 0.678\n",
      "399000 : 0.674\n",
      "400000 : 0.679\n",
      "401000 : 0.681\n",
      "402000 : 0.684\n",
      "403000 : 0.695\n",
      "404000 : 0.690\n",
      "405000 : 0.693\n",
      "406000 : 0.681\n",
      "407000 : 0.695\n",
      "408000 : 0.669\n",
      "409000 : 0.681\n",
      "410000 : 0.670\n",
      "411000 : 0.689\n",
      "412000 : 0.696\n",
      "413000 : 0.688\n",
      "414000 : 0.689\n",
      "415000 : 0.700\n",
      "416000 : 0.675\n",
      "417000 : 0.674\n",
      "418000 : 0.708\n",
      "419000 : 0.679\n",
      "420000 : 0.673\n",
      "421000 : 0.713\n",
      "422000 : 0.690\n",
      "423000 : 0.670\n",
      "424000 : 0.674\n",
      "425000 : 0.687\n",
      "426000 : 0.665\n",
      "427000 : 0.717\n",
      "428000 : 0.678\n",
      "429000 : 0.705\n",
      "430000 : 0.714\n",
      "431000 : 0.722\n",
      "432000 : 0.686\n",
      "433000 : 0.684\n",
      "434000 : 0.684\n",
      "435000 : 0.702\n",
      "436000 : 0.681\n",
      "437000 : 0.679\n",
      "438000 : 0.701\n",
      "439000 : 0.669\n",
      "440000 : 0.680\n",
      "441000 : 0.673\n",
      "442000 : 0.679\n",
      "443000 : 0.694\n",
      "444000 : 0.692\n",
      "445000 : 0.652\n",
      "446000 : 0.666\n",
      "447000 : 0.667\n",
      "448000 : 0.680\n",
      "449000 : 0.663\n",
      "450000 : 0.711\n",
      "451000 : 0.687\n",
      "452000 : 0.678\n",
      "453000 : 0.692\n",
      "454000 : 0.674\n",
      "455000 : 0.680\n",
      "456000 : 0.693\n",
      "457000 : 0.672\n",
      "458000 : 0.694\n",
      "459000 : 0.687\n",
      "460000 : 0.688\n",
      "461000 : 0.639\n",
      "462000 : 0.681\n",
      "463000 : 0.679\n",
      "464000 : 0.664\n",
      "465000 : 0.653\n",
      "466000 : 0.679\n",
      "467000 : 0.705\n",
      "468000 : 0.694\n",
      "469000 : 0.640\n",
      "470000 : 0.691\n",
      "471000 : 0.685\n",
      "472000 : 0.672\n",
      "473000 : 0.698\n",
      "474000 : 0.676\n",
      "475000 : 0.709\n",
      "476000 : 0.705\n",
      "477000 : 0.674\n",
      "478000 : 0.707\n",
      "479000 : 0.675\n",
      "480000 : 0.694\n",
      "481000 : 0.685\n",
      "482000 : 0.662\n",
      "483000 : 0.702\n",
      "484000 : 0.691\n",
      "485000 : 0.680\n",
      "486000 : 0.685\n",
      "487000 : 0.687\n",
      "488000 : 0.683\n",
      "489000 : 0.688\n",
      "490000 : 0.661\n",
      "491000 : 0.693\n",
      "492000 : 0.691\n",
      "493000 : 0.662\n",
      "494000 : 0.670\n",
      "495000 : 0.691\n",
      "496000 : 0.709\n",
      "497000 : 0.686\n",
      "498000 : 0.692\n",
      "499000 : 0.684\n",
      "500000 : 0.671\n",
      "501000 : 0.695\n",
      "502000 : 0.692\n",
      "503000 : 0.687\n",
      "504000 : 0.685\n",
      "505000 : 0.684\n",
      "506000 : 0.688\n",
      "507000 : 0.672\n",
      "508000 : 0.657\n",
      "509000 : 0.696\n",
      "510000 : 0.679\n",
      "511000 : 0.678\n",
      "512000 : 0.698\n",
      "513000 : 0.675\n",
      "514000 : 0.697\n",
      "515000 : 0.687\n",
      "516000 : 0.685\n",
      "517000 : 0.680\n",
      "518000 : 0.685\n",
      "519000 : 0.679\n",
      "520000 : 0.688\n",
      "521000 : 0.686\n",
      "522000 : 0.657\n",
      "523000 : 0.690\n",
      "524000 : 0.655\n",
      "525000 : 0.693\n",
      "526000 : 0.675\n",
      "527000 : 0.687\n",
      "528000 : 0.705\n",
      "529000 : 0.693\n",
      "530000 : 0.689\n",
      "531000 : 0.662\n",
      "532000 : 0.670\n",
      "533000 : 0.688\n",
      "534000 : 0.657\n",
      "535000 : 0.684\n",
      "536000 : 0.689\n",
      "537000 : 0.692\n",
      "538000 : 0.678\n",
      "539000 : 0.698\n",
      "540000 : 0.684\n",
      "541000 : 0.680\n",
      "542000 : 0.679\n",
      "543000 : 0.675\n",
      "544000 : 0.693\n",
      "545000 : 0.682\n",
      "546000 : 0.700\n",
      "547000 : 0.705\n",
      "548000 : 0.699\n",
      "549000 : 0.692\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "550000 : 0.683\n",
      "551000 : 0.701\n",
      "552000 : 0.711\n",
      "553000 : 0.655\n",
      "554000 : 0.694\n",
      "555000 : 0.661\n",
      "556000 : 0.665\n",
      "557000 : 0.696\n",
      "558000 : 0.678\n",
      "559000 : 0.694\n",
      "560000 : 0.677\n",
      "561000 : 0.684\n",
      "562000 : 0.657\n",
      "563000 : 0.688\n",
      "564000 : 0.681\n",
      "565000 : 0.650\n",
      "566000 : 0.667\n",
      "567000 : 0.689\n",
      "568000 : 0.702\n",
      "569000 : 0.649\n",
      "570000 : 0.707\n",
      "571000 : 0.688\n",
      "572000 : 0.669\n",
      "573000 : 0.690\n",
      "574000 : 0.712\n",
      "575000 : 0.680\n",
      "576000 : 0.707\n",
      "577000 : 0.693\n",
      "578000 : 0.690\n",
      "579000 : 0.673\n",
      "580000 : 0.672\n",
      "581000 : 0.679\n",
      "582000 : 0.694\n",
      "583000 : 0.660\n",
      "584000 : 0.677\n",
      "585000 : 0.673\n",
      "586000 : 0.704\n",
      "587000 : 0.664\n",
      "588000 : 0.685\n",
      "589000 : 0.699\n",
      "590000 : 0.659\n",
      "591000 : 0.672\n",
      "592000 : 0.689\n",
      "593000 : 0.701\n",
      "594000 : 0.680\n",
      "595000 : 0.693\n",
      "596000 : 0.691\n",
      "597000 : 0.664\n",
      "598000 : 0.682\n",
      "599000 : 0.685\n",
      "600000 : 0.685\n",
      "601000 : 0.684\n",
      "602000 : 0.706\n",
      "603000 : 0.665\n",
      "604000 : 0.673\n",
      "605000 : 0.705\n",
      "606000 : 0.673\n",
      "607000 : 0.697\n",
      "608000 : 0.687\n",
      "609000 : 0.683\n",
      "610000 : 0.669\n",
      "611000 : 0.680\n",
      "612000 : 0.692\n",
      "613000 : 0.691\n",
      "614000 : 0.695\n",
      "615000 : 0.679\n",
      "616000 : 0.687\n",
      "617000 : 0.699\n",
      "618000 : 0.698\n",
      "619000 : 0.701\n",
      "620000 : 0.687\n",
      "621000 : 0.679\n",
      "622000 : 0.674\n",
      "623000 : 0.687\n",
      "624000 : 0.685\n",
      "625000 : 0.682\n",
      "626000 : 0.711\n",
      "627000 : 0.675\n",
      "628000 : 0.683\n",
      "629000 : 0.713\n",
      "630000 : 0.699\n",
      "631000 : 0.695\n",
      "632000 : 0.687\n",
      "633000 : 0.708\n",
      "634000 : 0.692\n",
      "635000 : 0.679\n",
      "636000 : 0.673\n",
      "637000 : 0.691\n",
      "638000 : 0.680\n",
      "639000 : 0.674\n",
      "640000 : 0.707\n",
      "641000 : 0.690\n",
      "642000 : 0.679\n",
      "643000 : 0.689\n",
      "644000 : 0.706\n",
      "645000 : 0.682\n",
      "646000 : 0.686\n",
      "647000 : 0.684\n",
      "648000 : 0.690\n",
      "649000 : 0.712\n",
      "650000 : 0.643\n",
      "651000 : 0.715\n",
      "652000 : 0.700\n",
      "653000 : 0.703\n",
      "654000 : 0.676\n",
      "655000 : 0.658\n",
      "656000 : 0.690\n",
      "657000 : 0.693\n",
      "658000 : 0.691\n",
      "659000 : 0.705\n",
      "660000 : 0.666\n",
      "661000 : 0.692\n",
      "662000 : 0.676\n",
      "663000 : 0.706\n",
      "664000 : 0.691\n",
      "665000 : 0.689\n",
      "666000 : 0.671\n",
      "667000 : 0.697\n",
      "668000 : 0.685\n",
      "669000 : 0.708\n",
      "670000 : 0.689\n",
      "671000 : 0.689\n",
      "672000 : 0.666\n",
      "673000 : 0.702\n",
      "674000 : 0.712\n",
      "675000 : 0.692\n",
      "676000 : 0.709\n",
      "677000 : 0.673\n",
      "678000 : 0.704\n",
      "679000 : 0.722\n",
      "680000 : 0.668\n",
      "681000 : 0.669\n",
      "682000 : 0.657\n",
      "683000 : 0.676\n",
      "684000 : 0.675\n",
      "685000 : 0.686\n",
      "686000 : 0.694\n",
      "687000 : 0.675\n",
      "688000 : 0.679\n",
      "689000 : 0.675\n",
      "690000 : 0.676\n",
      "691000 : 0.676\n",
      "692000 : 0.692\n",
      "693000 : 0.682\n",
      "694000 : 0.657\n",
      "695000 : 0.680\n",
      "696000 : 0.683\n",
      "697000 : 0.692\n",
      "698000 : 0.688\n",
      "699000 : 0.697\n",
      "700000 : 0.692\n",
      "701000 : 0.669\n",
      "702000 : 0.687\n",
      "703000 : 0.698\n",
      "704000 : 0.675\n",
      "705000 : 0.645\n",
      "706000 : 0.697\n",
      "707000 : 0.665\n",
      "708000 : 0.680\n",
      "709000 : 0.691\n",
      "710000 : 0.680\n",
      "711000 : 0.702\n",
      "712000 : 0.672\n",
      "713000 : 0.697\n",
      "714000 : 0.695\n",
      "715000 : 0.674\n",
      "716000 : 0.684\n",
      "717000 : 0.680\n",
      "718000 : 0.687\n",
      "719000 : 0.695\n",
      "720000 : 0.702\n",
      "721000 : 0.687\n",
      "722000 : 0.686\n",
      "723000 : 0.686\n",
      "724000 : 0.710\n",
      "725000 : 0.693\n",
      "726000 : 0.678\n",
      "727000 : 0.677\n",
      "728000 : 0.666\n",
      "729000 : 0.698\n",
      "730000 : 0.699\n",
      "731000 : 0.703\n",
      "732000 : 0.696\n",
      "733000 : 0.714\n",
      "734000 : 0.693\n",
      "735000 : 0.709\n",
      "736000 : 0.662\n",
      "737000 : 0.670\n",
      "738000 : 0.688\n",
      "739000 : 0.678\n",
      "740000 : 0.674\n",
      "741000 : 0.680\n",
      "742000 : 0.694\n",
      "743000 : 0.676\n",
      "744000 : 0.661\n",
      "745000 : 0.678\n",
      "746000 : 0.664\n",
      "747000 : 0.675\n",
      "748000 : 0.678\n",
      "749000 : 0.681\n",
      "750000 : 0.695\n",
      "751000 : 0.669\n",
      "752000 : 0.672\n",
      "753000 : 0.688\n",
      "754000 : 0.682\n",
      "755000 : 0.693\n",
      "756000 : 0.696\n",
      "757000 : 0.694\n",
      "758000 : 0.679\n",
      "759000 : 0.658\n",
      "760000 : 0.725\n",
      "761000 : 0.694\n",
      "762000 : 0.681\n",
      "763000 : 0.668\n",
      "764000 : 0.677\n",
      "765000 : 0.707\n",
      "766000 : 0.691\n",
      "767000 : 0.706\n",
      "768000 : 0.680\n",
      "769000 : 0.686\n",
      "770000 : 0.664\n",
      "771000 : 0.677\n",
      "772000 : 0.698\n",
      "773000 : 0.690\n",
      "774000 : 0.688\n",
      "775000 : 0.695\n",
      "776000 : 0.682\n",
      "777000 : 0.707\n",
      "778000 : 0.690\n",
      "779000 : 0.682\n",
      "780000 : 0.678\n",
      "781000 : 0.677\n",
      "782000 : 0.716\n",
      "783000 : 0.682\n",
      "784000 : 0.685\n",
      "785000 : 0.703\n",
      "786000 : 0.689\n",
      "787000 : 0.702\n",
      "788000 : 0.683\n",
      "789000 : 0.675\n",
      "790000 : 0.667\n",
      "791000 : 0.691\n",
      "792000 : 0.675\n",
      "793000 : 0.672\n",
      "794000 : 0.694\n",
      "795000 : 0.692\n",
      "796000 : 0.676\n",
      "797000 : 0.686\n",
      "798000 : 0.691\n",
      "799000 : 0.691\n",
      "800000 : 0.673\n",
      "801000 : 0.677\n",
      "802000 : 0.678\n",
      "803000 : 0.678\n",
      "804000 : 0.661\n",
      "805000 : 0.687\n",
      "806000 : 0.677\n",
      "807000 : 0.648\n",
      "808000 : 0.685\n",
      "809000 : 0.711\n",
      "810000 : 0.655\n",
      "811000 : 0.682\n",
      "812000 : 0.702\n",
      "813000 : 0.693\n",
      "814000 : 0.670\n",
      "815000 : 0.690\n",
      "816000 : 0.643\n",
      "817000 : 0.721\n",
      "818000 : 0.684\n",
      "819000 : 0.700\n",
      "820000 : 0.689\n",
      "821000 : 0.685\n",
      "822000 : 0.684\n",
      "823000 : 0.688\n",
      "824000 : 0.703\n",
      "825000 : 0.671\n",
      "826000 : 0.701\n",
      "827000 : 0.693\n",
      "828000 : 0.713\n",
      "829000 : 0.684\n",
      "830000 : 0.682\n",
      "831000 : 0.655\n",
      "832000 : 0.679\n",
      "833000 : 0.663\n",
      "834000 : 0.709\n",
      "835000 : 0.684\n",
      "836000 : 0.702\n",
      "837000 : 0.684\n",
      "838000 : 0.671\n",
      "839000 : 0.690\n",
      "840000 : 0.680\n",
      "841000 : 0.674\n",
      "842000 : 0.690\n",
      "843000 : 0.690\n",
      "844000 : 0.695\n",
      "845000 : 0.680\n",
      "846000 : 0.675\n",
      "847000 : 0.685\n",
      "848000 : 0.684\n",
      "849000 : 0.663\n",
      "850000 : 0.668\n",
      "851000 : 0.702\n",
      "852000 : 0.671\n",
      "853000 : 0.680\n",
      "854000 : 0.695\n",
      "855000 : 0.696\n",
      "856000 : 0.670\n",
      "857000 : 0.657\n",
      "858000 : 0.684\n",
      "859000 : 0.694\n",
      "860000 : 0.706\n",
      "861000 : 0.682\n",
      "862000 : 0.686\n",
      "863000 : 0.690\n",
      "864000 : 0.687\n",
      "865000 : 0.688\n",
      "866000 : 0.706\n",
      "867000 : 0.674\n",
      "868000 : 0.671\n",
      "869000 : 0.670\n",
      "870000 : 0.690\n",
      "871000 : 0.656\n",
      "872000 : 0.687\n",
      "873000 : 0.636\n",
      "874000 : 0.673\n",
      "875000 : 0.702\n",
      "876000 : 0.695\n",
      "877000 : 0.665\n",
      "878000 : 0.705\n",
      "879000 : 0.697\n",
      "880000 : 0.686\n",
      "881000 : 0.697\n",
      "882000 : 0.687\n",
      "883000 : 0.678\n",
      "884000 : 0.684\n",
      "885000 : 0.672\n",
      "886000 : 0.669\n",
      "887000 : 0.695\n",
      "888000 : 0.694\n",
      "889000 : 0.686\n",
      "890000 : 0.653\n",
      "891000 : 0.669\n",
      "892000 : 0.706\n",
      "893000 : 0.675\n",
      "894000 : 0.690\n",
      "895000 : 0.680\n",
      "896000 : 0.703\n",
      "897000 : 0.685\n",
      "898000 : 0.689\n",
      "899000 : 0.687\n",
      "900000 : 0.690\n",
      "901000 : 0.672\n",
      "902000 : 0.669\n",
      "903000 : 0.694\n",
      "904000 : 0.693\n",
      "905000 : 0.680\n",
      "906000 : 0.684\n",
      "907000 : 0.706\n",
      "908000 : 0.648\n",
      "909000 : 0.680\n",
      "910000 : 0.691\n",
      "911000 : 0.661\n",
      "912000 : 0.689\n",
      "913000 : 0.672\n",
      "914000 : 0.675\n",
      "915000 : 0.688\n",
      "916000 : 0.670\n",
      "917000 : 0.709\n",
      "918000 : 0.669\n",
      "919000 : 0.695\n",
      "920000 : 0.666\n",
      "921000 : 0.680\n",
      "922000 : 0.688\n",
      "923000 : 0.716\n",
      "924000 : 0.687\n",
      "925000 : 0.684\n",
      "926000 : 0.687\n",
      "927000 : 0.685\n",
      "928000 : 0.681\n",
      "929000 : 0.699\n",
      "930000 : 0.679\n",
      "931000 : 0.665\n",
      "932000 : 0.677\n",
      "933000 : 0.714\n",
      "934000 : 0.659\n",
      "935000 : 0.656\n",
      "936000 : 0.686\n",
      "937000 : 0.697\n",
      "938000 : 0.696\n",
      "939000 : 0.682\n",
      "940000 : 0.667\n",
      "941000 : 0.680\n",
      "942000 : 0.681\n",
      "943000 : 0.668\n",
      "944000 : 0.677\n",
      "945000 : 0.703\n",
      "946000 : 0.653\n",
      "947000 : 0.694\n",
      "948000 : 0.684\n",
      "949000 : 0.691\n",
      "950000 : 0.670\n",
      "951000 : 0.698\n",
      "952000 : 0.695\n",
      "953000 : 0.649\n",
      "954000 : 0.680\n",
      "955000 : 0.700\n",
      "956000 : 0.660\n",
      "957000 : 0.686\n",
      "958000 : 0.698\n",
      "959000 : 0.701\n",
      "960000 : 0.704\n",
      "961000 : 0.684\n",
      "962000 : 0.684\n",
      "963000 : 0.700\n",
      "964000 : 0.689\n",
      "965000 : 0.674\n",
      "966000 : 0.720\n",
      "967000 : 0.701\n",
      "968000 : 0.696\n",
      "969000 : 0.678\n",
      "970000 : 0.664\n",
      "971000 : 0.690\n",
      "972000 : 0.712\n",
      "973000 : 0.690\n",
      "974000 : 0.679\n",
      "975000 : 0.682\n",
      "976000 : 0.692\n",
      "977000 : 0.684\n",
      "978000 : 0.663\n",
      "979000 : 0.697\n",
      "980000 : 0.664\n",
      "981000 : 0.680\n",
      "982000 : 0.702\n",
      "983000 : 0.684\n",
      "984000 : 0.673\n",
      "985000 : 0.660\n",
      "986000 : 0.685\n",
      "987000 : 0.702\n",
      "988000 : 0.709\n",
      "989000 : 0.686\n",
      "990000 : 0.675\n",
      "991000 : 0.699\n",
      "992000 : 0.695\n",
      "993000 : 0.668\n",
      "994000 : 0.672\n",
      "995000 : 0.697\n",
      "996000 : 0.665\n",
      "997000 : 0.694\n",
      "998000 : 0.687\n",
      "999000 : 0.665\n",
      "1000000 : 0.688\n"
     ]
    }
   ],
   "source": [
    "# Calculate and print the average reward per thousand episodes\n",
    "rewards_per_thosand_episodes = np.split(np.array(rewards_all_episode), num_episodes/1000)\n",
    "count = 1000\n",
    "\n",
    "print(\"********Average reward per thousand episodes********\\n\")\n",
    "for r in rewards_per_thosand_episodes:\n",
    "    print(count, \": %.3f\" % (sum(r/1000)))\n",
    "    count += 1000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.52960222, 0.50696265, 0.50504294, 0.49757901],\n",
       "       [0.31466148, 0.3183473 , 0.30041897, 0.44279878],\n",
       "       [0.37440354, 0.3143898 , 0.28653805, 0.33001418],\n",
       "       [0.17500452, 0.02335419, 0.01221098, 0.02994933],\n",
       "       [0.54795294, 0.36492849, 0.37542263, 0.35341444],\n",
       "       [0.        , 0.        , 0.        , 0.        ],\n",
       "       [0.27228914, 0.188166  , 0.33530807, 0.11939158],\n",
       "       [0.        , 0.        , 0.        , 0.        ],\n",
       "       [0.35140246, 0.40766645, 0.36918665, 0.57685318],\n",
       "       [0.44996111, 0.62552095, 0.45145697, 0.36918563],\n",
       "       [0.59748698, 0.48457239, 0.40913522, 0.32257495],\n",
       "       [0.        , 0.        , 0.        , 0.        ],\n",
       "       [0.        , 0.        , 0.        , 0.        ],\n",
       "       [0.43515616, 0.50202965, 0.73987762, 0.47832563],\n",
       "       [0.72828018, 0.8552339 , 0.80261344, 0.77744184],\n",
       "       [0.        , 0.        , 0.        , 0.        ]])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Interpreting the results\n",
    "q_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (Right)\n",
      "SFFF\n",
      "FHFH\n",
      "FFFH\n",
      "HFFG\n",
      "Congratulations, the agent has reach the Goal in 51 steps\n"
     ]
    }
   ],
   "source": [
    "# watch our agent play the frozen lake by playing the best action\n",
    "for episode in range(3):\n",
    "    # initialize the new episode\n",
    "    done= False\n",
    "    state= env.reset()\n",
    "    for step in range(max_steps_per_episode):\n",
    "        # show current step\n",
    "        # choose action and apply\n",
    "        clear_output(wait= True)\n",
    "        env.render()\n",
    "        time.sleep(0.1)\n",
    "        action= np.argmax(q_table[state, :])\n",
    "        new_state, reward, done, info= env.step(action)      \n",
    "        \n",
    "        if step == max_steps_per_episode-1:\n",
    "            print('The Agent is dumb and haven\\'t figured out in % steps' % (max_steps_per_episode))\n",
    "\n",
    "        if done:\n",
    "            if reward == 1:\n",
    "                print('Congratulations, the agent has reach the Goal in %d steps' % (step))\n",
    "                time.sleep(5)\n",
    "            else:\n",
    "                print('Oh, the agent fell into the lake in %d steps' % (step))\n",
    "                time.sleep(5)\n",
    "            break\n",
    "        # set new state\n",
    "        state= new_state\n",
    "    clear_output(wait= True)\n",
    "env.close()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 2
}