Setup GridWorld.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Implementation of GridWorld\n",
    "\n",
    "As described in Chapter 3 of Sutton & Barto Book on Reinforcement Learning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from gamelearner import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GridWorldGame:\n",
    "    \"\"\"Simulates a game of Gridworld.\n",
    "\n",
    "    Class attributes:\n",
    "        GridWorldGame.name (str): The game's name ('Tic Tac Toe').\n",
    "        GridWorldGame.size (int): Width (and height) of grid.\n",
    "        roles [int]: The player role (there is only one).\n",
    "        GridWorldGame.possible_n_players (list): List of allowed\n",
    "            numbers of players ([1]).\n",
    "        TicTacToeGame.marks (list): The characters used to represent\n",
    "            each role's move on the board (['X']).\n",
    "        TicTacToeGame.help_text (dict): Various messages (strings)\n",
    "            to help user.\n",
    "    \"\"\"\n",
    "\n",
    "    name = 'Grid World'\n",
    "    SIZE = 5\n",
    "    default_shape = (SIZE, SIZE)\n",
    "    roles = [1]\n",
    "    possible_n_players = [1]\n",
    "    marks = ['X']\n",
    "\n",
    "    help_text = {\n",
    "        'Move format': \"l, r, u, or d\",\n",
    "        'Move not available': \"That move is not possible.\",\n",
    "        'Number of players': \"This game is only for 1 player.\"\n",
    "    }\n",
    "\n",
    "    A_POS = (0, 1)\n",
    "    A_PRIME_POS = (4, 1)\n",
    "    B_POS = (0, 3)\n",
    "    B_PRIME_POS = (2, 3)\n",
    "\n",
    "    def __init__(self, start_pos=(0, 0), moves=None, shape=default_shape,\n",
    "                 a_pos=A_POS, a_prime_pos=A_PRIME_POS, b_pos=B_POS, \n",
    "                 b_prime_pos=B_PRIME_POS, discount=0.9):\n",
    "        \"\"\"Initialize a game.\n",
    "\n",
    "        Args:\n",
    "            moves (list): This is optional. Provide a list of completed\n",
    "                moves. Each move should be a char indicating the move \n",
    "                ('l', 'r', 'u', 'd').\n",
    "        \"\"\"\n",
    "        \n",
    "        self.n_players = 1\n",
    "        self.start_pos = start_pos\n",
    "        self.shape = shape\n",
    "        self.a_pos = a_pos\n",
    "        self.a_prime_pos = a_prime_pos\n",
    "        self.b_pos = b_pos\n",
    "        self.b_prime_pos = b_prime_pos\n",
    "        self.actions = {\n",
    "            'l': np.array([0, -1]),\n",
    "            'u': np.array([-1, 0]),\n",
    "            'r': np.array([0, 1]),\n",
    "            'd': np.array([1, 0])\n",
    "        }\n",
    "        self.reverse_actions = {\n",
    "            'l': 'r',\n",
    "            'u': 'd',\n",
    "            'r': 'l',\n",
    "            'd': 'u'           \n",
    "        }\n",
    "        self.discount = discount\n",
    "        self.start_time = None\n",
    "        self.end_time = None\n",
    "        self.game_over = False\n",
    "        self.reset()\n",
    "        if moves is not None:\n",
    "            for move in moves:\n",
    "                self.make_move(move)\n",
    "            self.start()\n",
    "\n",
    "    def start(self):\n",
    "        \"\"\"Record start time (self.start_time).\"\"\"\n",
    "\n",
    "        self.start_time = datetime.datetime.now()\n",
    "\n",
    "    def stop(self):\n",
    "        \"\"\"Record end time (self.end_time).\"\"\"\n",
    "\n",
    "        self.end_time = datetime.datetime.now()\n",
    "            \n",
    "    def reset(self):\n",
    "        \"\"\"Set the state of the game to the beginning (no moves).\n",
    "        \"\"\"\n",
    "\n",
    "        self.moves = []\n",
    "        self.state = np.array(self.start_pos)\n",
    "        self.game_over = False\n",
    "        self.start_time = None\n",
    "        self.end_time = None\n",
    "        self.turn = self.roles[0]\n",
    "\n",
    "    def show_state(self):\n",
    "        \"\"\"Display the current state of the gridworld.\"\"\"\n",
    "\n",
    "        chars = '_abAB' + self.marks[0]\n",
    "        \n",
    "        states = np.zeros(self.shape, dtype='b')\n",
    "        states[self.a_pos] = 1\n",
    "        states[self.b_pos] = 2\n",
    "        states[self.a_prime_pos] = 3\n",
    "        states[self.b_prime_pos] = 4\n",
    "        states[tuple(self.state)] = 5\n",
    "        \n",
    "        for row in states:\n",
    "            print(\" \".join(list(chars[i] for i in row)))\n",
    "\n",
    "    def available_moves(self, state=None):\n",
    "        \"\"\"Returns list of available moves ('l', 'r', 'u', 'd').\n",
    "        \"\"\"\n",
    "\n",
    "        return list(self.actions.keys())\n",
    "\n",
    "    def update_state(self, move, state=None, set_rewards=False):\n",
    "        \"\"\"Updates the game state with the move to be taken.\n",
    "\n",
    "        Args:\n",
    "            move (string): Player's move ('l', 'r', 'u', or 'd').\n",
    "            state (np.ndarray): Array (size (3, 3)) of game state or if\n",
    "                not provided the current game state will be used.\n",
    "            set_rewards (bool): If True, self.rewards will be set after\n",
    "                based on the move made.\n",
    "\n",
    "        Raises:\n",
    "            ValueError if the position is out of bounds or if\n",
    "            there is already a move in that position.\n",
    "        \"\"\"\n",
    "\n",
    "        if state is None:\n",
    "            state = self.state\n",
    "\n",
    "        if np.all(self.state == self.a_pos):\n",
    "            state[:] = self.a_prime_pos\n",
    "            reward = 10.0\n",
    "        elif np.all(self.state == self.b_pos):\n",
    "            state[:] = self.b_prime_pos\n",
    "            reward = 5.0\n",
    "        else:\n",
    "            state += self.actions[move[1]]\n",
    "            x, y = state\n",
    "            if x < 0 or x >= self.shape[0] or y < 0 or y >= self.shape[1]:\n",
    "                reward = -1.0\n",
    "                state -= self.actions[move[1]]\n",
    "            else:\n",
    "                reward = 0.0\n",
    "        \n",
    "        if set_rewards:\n",
    "            self.reward = reward\n",
    "\n",
    "    def next_state(self, move, state=None):\n",
    "        \"\"\"Returns the next state of the game if move were to be\n",
    "        taken from current game state or from state if provided.\n",
    "\n",
    "        Args:\n",
    "            move (tuple): Tuple containing player role (1) and a move\n",
    "                ('l', 'r', 'u', or 'd').\n",
    "            state (np.ndarray): Array (size (2,)) of game state or if\n",
    "                                not provided the current game state will\n",
    "                                be used.\n",
    "\n",
    "        Returns:\n",
    "            next_state (np.ndarray): copy of state after move made.\n",
    "        \"\"\"\n",
    "\n",
    "        if state is None:\n",
    "            state = self.state\n",
    "        next_state = state.copy()\n",
    "        self.update_state(move, state=next_state)\n",
    "\n",
    "        return next_state\n",
    "\n",
    "    def make_move(self, move, show=False):\n",
    "        \"\"\"Update the world with a new move.\n",
    "\n",
    "        Args:\n",
    "            move (tuple): Tuple containing player role (1) and a move\n",
    "                ('l', 'r', 'u', or 'd').\n",
    "            show (bool): Print a message if True.\n",
    "        \"\"\"\n",
    "\n",
    "        if move[1] not in self.available_moves():\n",
    "            raise ValueError(self.help_text['Move not available'])\n",
    "\n",
    "        self.update_state(move, set_rewards=True)\n",
    "        self.moves.append(move)\n",
    "        if show:\n",
    "            print(\"Player made move %s\" % move)\n",
    "\n",
    "    def reverse_move(self, show=False):\n",
    "        \"\"\"Reverse the last move made.\n",
    "\n",
    "        Args:\n",
    "            show (bool): Print a message if True.\n",
    "        \"\"\"\n",
    "\n",
    "        # TODO: This reverses the state but does not restore the \n",
    "        # previous reward\n",
    "        last_move = self.moves.pop()\n",
    "        if np.all(self.state == self.a_prime_pos):\n",
    "            self.state[:] = self.a_pos\n",
    "        elif np.all(self.state == self.b_prime_pos):\n",
    "            self.state[:] = self.b_pos\n",
    "        else:\n",
    "            reverse_move = (last_move[0], self.reverse_actions[last_move[1]])\n",
    "            self.state += self.actions[reverse_move[1]]\n",
    "\n",
    "        if show:\n",
    "            print(\"Last move reversed\")\n",
    "\n",
    "    def get_rewards(self):\n",
    "        \"\"\"Returns the reward for the player at the current time step\n",
    "        as a dictionary.\n",
    "        \"\"\"\n",
    "\n",
    "        return {self.turn: self.reward}\n",
    "\n",
    "    def generate_action_key(self, state, move):\n",
    "        \"\"\"Converts a game state in the form of an array into an\n",
    "        integer.  This is used by TDLearner to create unique \n",
    "        hashable keys for storing values in a dictionary.\n",
    "\n",
    "        Args:\n",
    "            state (np.ndarray): Game state array (shape may depend\n",
    "                on the game) of type int.\n",
    "            move (role, action): Tuple containing player role\n",
    "                and move ('l', 'r', 'u', or 'd').\n",
    "\n",
    "        Returns:\n",
    "            key (int): unique integer representation of game state.\n",
    "        \"\"\"\n",
    "\n",
    "        assert 0 <= state[0] < game.shape[0]\n",
    "        assert 0 <= state[1] < game.shape[1]\n",
    "\n",
    "        role, action = move\n",
    "        assert action in self.actions.keys()\n",
    "\n",
    "        return tuple(state), action\n",
    "\n",
    "    def __repr__(self):\n",
    "\n",
    "            params = []\n",
    "            if self.start_pos != (0, 0):\n",
    "                params.append(\"start_pos=%s\" % self.start_pos.__repr__())\n",
    "            if self.moves:\n",
    "                params.append(\"moves=%s\" % self.moves.__repr__())\n",
    "            if self.shape != self.default_shape:\n",
    "                params.append(\"shape=%s\" % self.shape.__repr__())\n",
    "            return \"GridWorldGame(%s)\" % ', '.join(params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1], dtype=int8)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "state = np.array([1, 1], dtype='b')\n",
    "state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridWorldGame()"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game = GridWorldGame()\n",
    "game"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([1], 1)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.roles, game.turn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X a _ b _\n",
      "_ _ _ _ _\n",
      "_ _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n"
     ]
    }
   ],
   "source": [
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['l', 'u', 'r', 'd']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.available_moves()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.make_move((1, 'd'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 0.0}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.get_rewards()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_ a _ b _\n",
      "X _ _ _ _\n",
      "_ _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n"
     ]
    }
   ],
   "source": [
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1, 0), 'r')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.generate_action_key(game.state, (1, 'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_ a _ b _\n",
      "X _ _ _ _\n",
      "_ _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n"
     ]
    }
   ],
   "source": [
    "game.make_move((1, 'l'))\n",
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: -1.0}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.get_rewards()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_ a _ b _\n",
      "_ X _ _ _\n",
      "_ _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n"
     ]
    }
   ],
   "source": [
    "game.make_move((1, 'r'))\n",
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_ X _ b _\n",
      "_ _ _ _ _\n",
      "_ _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n"
     ]
    }
   ],
   "source": [
    "game.make_move((1, 'u'))\n",
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_ a _ b _\n",
      "_ _ _ _ _\n",
      "_ _ _ B _\n",
      "_ _ _ _ _\n",
      "_ X _ _ _\n"
     ]
    }
   ],
   "source": [
    "random_move = np.random.choice(list(game.actions.keys()))\n",
    "game.make_move((1, random_move))\n",
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 10.0}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.get_rewards()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, 'd'), (1, 'l'), (1, 'r'), (1, 'u'), (1, 'l')]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.moves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_ a _ b _\n",
      "_ _ _ _ _\n",
      "_ _ _ B _\n",
      "_ X _ _ _\n",
      "_ A _ _ _\n"
     ]
    }
   ],
   "source": [
    "game.make_move((1, 'u'))\n",
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_ a _ b _\n",
      "_ _ _ _ _\n",
      "_ _ _ B _\n",
      "_ _ _ _ _\n",
      "_ X _ _ _\n"
     ]
    }
   ],
   "source": [
    "game.reverse_move()\n",
    "game.show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(1, 'd'), (1, 'l'), (1, 'r'), (1, 'u'), (1, 'l')]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "game.moves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X a _ b _\n",
      "_ _ _ _ _\n",
      "_ _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n"
     ]
    }
   ],
   "source": [
    "game.show_state()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Play a game with a computer player"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This game is only for 1 player.\n"
     ]
    }
   ],
   "source": [
    "players = [TDLearner(\"TD1\"), TDLearner(\"TD2\")]\n",
    "try:\n",
    "    ctrl = GameController(game, players)\n",
    "except AssertionError as err:\n",
    "    print(err)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "td = TDLearner(\"TD1\")\n",
    "players = [td]\n",
    "ctrl = GameController(game, players)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Game of Grid World with 1 players ['TD1']\n",
      "X a _ b _\n",
      "_ _ _ _ _\n",
      "_ _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n",
      "TD1's turn (l, r, u, or d): d\n",
      "_ a _ b _\n",
      "X _ _ _ _\n",
      "_ _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n",
      "TD1's turn (l, r, u, or d): d\n",
      "_ a _ b _\n",
      "_ _ _ _ _\n",
      "X _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n",
      "TD1's turn (l, r, u, or d): l\n",
      "_ a _ b _\n",
      "_ _ _ _ _\n",
      "X _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n",
      "TD1's turn (l, r, u, or d): l\n",
      "_ a _ b _\n",
      "_ _ _ _ _\n",
      "X _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n",
      "TD1's turn (l, r, u, or d): r\n",
      "_ a _ b _\n",
      "_ _ _ _ _\n",
      "_ X _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n",
      "TD1's turn (l, r, u, or d): l\n",
      "_ a _ b _\n",
      "_ _ _ _ _\n",
      "X _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n",
      "TD1's turn (l, r, u, or d): d\n",
      "_ a _ b _\n",
      "_ _ _ _ _\n",
      "_ _ _ B _\n",
      "X _ _ _ _\n",
      "_ A _ _ _\n",
      "TD1's turn (l, r, u, or d): l\n",
      "_ a _ b _\n",
      "_ _ _ _ _\n",
      "_ _ _ B _\n",
      "X _ _ _ _\n",
      "_ A _ _ _\n",
      "TD1's turn (l, r, u, or d): u\n",
      "_ a _ b _\n",
      "_ _ _ _ _\n",
      "X _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n",
      "TD1's turn (l, r, u, or d): r\n"
     ]
    }
   ],
   "source": [
    "ctrl.play(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{((0, 0), 'l'): 0.5,\n",
       " ((0, 0), 'u'): 0.5,\n",
       " ((0, 0), 'r'): 0.5,\n",
       " ((0, 0), 'd'): 0.5,\n",
       " ((1, 0), 'l'): 0.5,\n",
       " ((1, 0), 'u'): 0.5,\n",
       " ((1, 0), 'r'): 0.5,\n",
       " ((1, 0), 'd'): -0.5,\n",
       " ((2, 0), 'l'): -0.25,\n",
       " ((2, 0), 'u'): 0.5,\n",
       " ((2, 0), 'r'): 0.5,\n",
       " ((2, 0), 'd'): -0.5,\n",
       " ((2, 1), 'l'): 0.5,\n",
       " ((2, 1), 'u'): 0.5,\n",
       " ((2, 1), 'r'): 0.5,\n",
       " ((2, 1), 'd'): 0.5,\n",
       " ((3, 0), 'l'): 0.5,\n",
       " ((3, 0), 'u'): 0.5,\n",
       " ((3, 0), 'r'): 0.5,\n",
       " ((3, 0), 'd'): 0.5}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "td.value_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[((0, 0), 'd'),\n",
       " ((1, 0), 'd'),\n",
       " ((2, 0), 'l'),\n",
       " ((2, 0), 'l'),\n",
       " ((2, 0), 'r'),\n",
       " ((2, 1), 'l'),\n",
       " ((2, 0), 'd'),\n",
       " ((3, 0), 'l'),\n",
       " ((3, 0), 'u'),\n",
       " ((2, 0), 'r')]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(td.saved_game_actions.values())[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Average rewards in each position"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 7.25 10.00 -0.25  5.00 -0.50 \n",
      "-0.25  0.00  0.00  0.00 -0.25 \n",
      "-0.25  0.00  1.25  0.00  3.75 \n",
      "-0.25  0.00  0.00  2.50 -0.25 \n",
      "-0.50 -0.25  7.50 -0.25 -0.50 \n"
     ]
    }
   ],
   "source": [
    "values = np.zeros(game.shape[0]*game.shape[1]).reshape(game.shape)\n",
    "role = 1\n",
    "\n",
    "for row in range(game.shape[0]):\n",
    "    for col in range(game.shape[1]):\n",
    "        game = GridWorldGame(start_pos=(row, col))\n",
    "        rewards = []\n",
    "        for a in game.available_moves():\n",
    "            game.make_move((role, a))\n",
    "            rewards.append(game.get_rewards()[role])\n",
    "            game.reverse_move()\n",
    "        assert len(rewards) == 4\n",
    "        values[row, col] = sum(rewards)/len(rewards)\n",
    "\n",
    "for row in values:\n",
    "    print(\"%5.2f \"*len(row) % tuple(row))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find discounted values of each position by random movement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "td = TDLearner(learning_rate=0.25, off_policy_rate=1.0)  # always random moves\n",
    "role = game.roles[0]\n",
    "\n",
    "for i in range(15000):\n",
    "    random_start_pos = tuple(random.randint(0, s-1) for s in game.shape)\n",
    "    game = GridWorldGame(start_pos=random_start_pos)\n",
    "    td.make_move(game=game, role=role)\n",
    "    td.make_move(game=game, role=role)  # Only get reward on second move\n",
    "    reward = game.get_rewards()[role]\n",
    "    td.on_policy = True   # Fool it so that it updates value function\n",
    "    td.reward(game=game, role=role, reward=reward)\n",
    "    if i == 5000:\n",
    "        td.learning_rate = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(((2, 3), 'd'), -3.551507096538684),\n",
       " (((3, 3), 'l'), -5.8193113737861495),\n",
       " (((2, 3), 'l'), 10.428411354510283),\n",
       " (((2, 2), 'r'), 10.421755484956817),\n",
       " (((4, 4), 'd'), -24.2977437991157),\n",
       " (((2, 4), 'l'), 4.207747438584605),\n",
       " (((2, 3), 'r'), -1.2410274912674102),\n",
       " (((0, 3), 'r'), 7.560383047216166),\n",
       " (((0, 3), 'u'), 9.039087834897899),\n",
       " (((1, 4), 'd'), -9.159707457392146)]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(td.value_function.items())[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 100)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(td.value_function), game.shape[0]*game.shape[1]*len(game.actions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 86.4 -12.2  85.6   9.0  54.3 \n",
      " 43.9  86.3  60.0  55.4  26.1 \n",
      " 22.5  25.4  32.5  26.8   9.2 \n",
      "  5.5  11.0  15.2  11.4   0.6 \n",
      " -7.9   0.4  -4.6  -7.7 -14.8 \n"
     ]
    }
   ],
   "source": [
    "values = np.zeros(game.shape[0]*game.shape[1]).reshape(game.shape)\n",
    "\n",
    "for row in range(game.shape[0]):\n",
    "    for col in range(game.shape[1]):\n",
    "        x = [td.get_value(((row, col), a)) for a in game.actions.keys()]\n",
    "        max_value = max(x)\n",
    "        values[row, col] = max_value\n",
    "\n",
    "for row in values:\n",
    "    print(\"%5.1f \"*len(row) % tuple(row))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X a _ b _\n",
      "_ _ _ _ _\n",
      "_ _ _ B _\n",
      "_ _ _ _ _\n",
      "_ A _ _ _\n"
     ]
    }
   ],
   "source": [
    "GridWorldGame().show_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}