SARSA(lambda)_learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "\n",
    "import asyncio\n",
    "import os\n",
    "import re\n",
    "from datetime import date\n",
    "\n",
    "import json\n",
    "import matplotlib\n",
    "import neptune.new as neptune\n",
    "import nest_asyncio\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "\n",
    "from collections import defaultdict\n",
    "from itertools import product\n",
    "from matplotlib import pyplot\n",
    "from poke_env.environment.abstract_battle import AbstractBattle\n",
    "from poke_env.player.battle_order import ForfeitBattleOrder\n",
    "from poke_env.player.player import Player\n",
    "# from poke_env.player.random_player import RandomPlayer\n",
    "from scipy.interpolate import griddata\n",
    "from src.playerMC import Player as PlayerSarsa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# global configs\n",
    "\n",
    "debug = True\n",
    "save_to_json_file = True\n",
    "use_validation = False\n",
    "\n",
    "nest_asyncio.apply()\n",
    "np.random.seed(0)\n",
    "\n",
    "use_neptune = True\n",
    "if use_neptune:\n",
    "    run = neptune.init(project='leolellisr/rl-pokeenv',\n",
    "                       api_token='eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLm5lcHR1bmUuYWkiLCJhcGlfdXJsIjoiaHR0cHM6Ly9hcHAubmVwdHVuZS5haSIsImFwaV9rZXkiOiI1NjY1YmJkZi1hYmM5LTQ3M2QtOGU1ZC1iZTFlNWY4NjE1NDQifQ==',\n",
    "                       name= 'SarsaDeterministic', tags=['Bruno', 'Sarsa', 'Deterministic', 'Train'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# our team\n",
    "\n",
    "OUR_TEAM = \"\"\"\n",
    "Pikachu-Original (M) @ Light Ball  \n",
    "Ability: Static  \n",
    "EVs: 252 Atk / 4 SpD / 252 Spe  \n",
    "Jolly Nature  \n",
    "- Volt Tackle  \n",
    "- Nuzzle  \n",
    "- Iron Tail  \n",
    "- Knock Off  \n",
    "\n",
    "Charizard @ Life Orb  \n",
    "Ability: Solar Power  \n",
    "EVs: 252 SpA / 4 SpD / 252 Spe  \n",
    "Timid Nature  \n",
    "IVs: 0 Atk  \n",
    "- Flamethrower  \n",
    "- Dragon Pulse  \n",
    "- Roost  \n",
    "- Sunny Day  \n",
    "\n",
    "Blastoise @ White Herb  \n",
    "Ability: Torrent  \n",
    "EVs: 4 Atk / 252 SpA / 252 Spe  \n",
    "Mild Nature  \n",
    "- Scald  \n",
    "- Ice Beam  \n",
    "- Earthquake  \n",
    "- Shell Smash  \n",
    "\n",
    "Venusaur @ Black Sludge  \n",
    "Ability: Chlorophyll  \n",
    "EVs: 252 SpA / 4 SpD / 252 Spe  \n",
    "Modest Nature  \n",
    "IVs: 0 Atk  \n",
    "- Giga Drain  \n",
    "- Sludge Bomb  \n",
    "- Sleep Powder  \n",
    "- Leech Seed  \n",
    "\n",
    "Sirfetch’d @ Aguav Berry  \n",
    "Ability: Steadfast  \n",
    "EVs: 248 HP / 252 Atk / 8 SpD  \n",
    "Adamant Nature  \n",
    "- Close Combat  \n",
    "- Swords Dance  \n",
    "- Poison Jab  \n",
    "- Knock Off  \n",
    "\n",
    "Tauros (M) @ Assault Vest  \n",
    "Ability: Intimidate  \n",
    "EVs: 252 Atk / 4 SpD / 252 Spe  \n",
    "Jolly Nature  \n",
    "- Double-Edge  \n",
    "- Earthquake  \n",
    "- Megahorn  \n",
    "- Iron Head  \n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# opponent's team\n",
    "\n",
    "OP_TEAM = \"\"\"\n",
    "Eevee @ Eviolite  \n",
    "Ability: Adaptability  \n",
    "EVs: 252 HP / 252 Atk / 4 SpD  \n",
    "Adamant Nature  \n",
    "- Quick Attack  \n",
    "- Flail  \n",
    "- Facade  \n",
    "- Wish  \n",
    "\n",
    "Vaporeon @ Leftovers  \n",
    "Ability: Hydration  \n",
    "EVs: 252 HP / 252 Def / 4 SpA  \n",
    "Bold Nature  \n",
    "IVs: 0 Atk  \n",
    "- Scald  \n",
    "- Shadow Ball  \n",
    "- Toxic  \n",
    "- Wish  \n",
    "\n",
    "Sylveon @ Aguav Berry  \n",
    "Ability: Pixilate  \n",
    "EVs: 252 HP / 252 SpA / 4 SpD  \n",
    "Modest Nature  \n",
    "IVs: 0 Atk  \n",
    "- Hyper Voice  \n",
    "- Mystical Fire  \n",
    "- Psyshock  \n",
    "- Calm Mind  \n",
    "\n",
    "Jolteon @ Assault Vest  \n",
    "Ability: Quick Feet  \n",
    "EVs: 252 SpA / 4 SpD / 252 Spe  \n",
    "Timid Nature  \n",
    "IVs: 0 Atk  \n",
    "- Thunderbolt  \n",
    "- Hyper Voice  \n",
    "- Volt Switch  \n",
    "- Shadow Ball  \n",
    "\n",
    "Leafeon @ Life Orb  \n",
    "Ability: Chlorophyll  \n",
    "EVs: 252 Atk / 4 SpD / 252 Spe  \n",
    "Adamant Nature  \n",
    "- Leaf Blade  \n",
    "- Knock Off  \n",
    "- X-Scissor  \n",
    "- Swords Dance  \n",
    "\n",
    "Umbreon @ Iapapa Berry  \n",
    "Ability: Inner Focus  \n",
    "EVs: 252 HP / 4 Atk / 252 SpD  \n",
    "Careful Nature  \n",
    "- Foul Play  \n",
    "- Body Slam  \n",
    "- Toxic  \n",
    "- Wish  \n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_OUR_MOVE_ACTIONS = 4\n",
    "N_OUR_SWITCH_ACTIONS = 5\n",
    "N_OUR_ACTIONS = N_OUR_MOVE_ACTIONS + N_OUR_SWITCH_ACTIONS\n",
    "\n",
    "ALL_OUR_ACTIONS = np.array(range(0, N_OUR_ACTIONS))\n",
    "\n",
    "NAME_TO_ID_DICT = {\n",
    "    \"pikachuoriginal\": 0,\n",
    "    \"charizard\": 1,\n",
    "    \"blastoise\": 2,\n",
    "    \"venusaur\": 3,\n",
    "    \"sirfetchd\": 4,\n",
    "    \"tauros\": 5,\n",
    "    \"eevee\": 6,\n",
    "    \"vaporeon\": 7,\n",
    "    \"sylveon\": 8,\n",
    "    \"jolteon\": 9,\n",
    "    \"leafeon\": 10,\n",
    "    \"umbreon\": 11\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Max-damage player\n",
    "\n",
    "class MaxDamagePlayer(Player):\n",
    "    def choose_move(self, battle):\n",
    "        if battle.available_moves:\n",
    "            best_move = max(battle.available_moves, key=lambda move: move.base_power)\n",
    "            return self.create_order(best_move)\n",
    "        else:\n",
    "            return self.choose_random_move(battle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# SARSA player\n",
    "class SARSAPlayer(PlayerSarsa):\n",
    "    def __init__(self, battle_format, team, n0, gamma, lambda_):\n",
    "        super().__init__(battle_format=battle_format, team=team)\n",
    "        self.N = defaultdict(lambda: np.zeros(N_OUR_ACTIONS))\n",
    "        self.Q = defaultdict(lambda: np.zeros(N_OUR_ACTIONS))\n",
    "        self.E = defaultdict(lambda: np.zeros(N_OUR_ACTIONS))\n",
    "        self.n0 = n0\n",
    "        self.gamma = gamma\n",
    "        self.state = None\n",
    "        self.action = None\n",
    "        self.lambda_ = lambda_\n",
    "\n",
    "    def choose_move(self, battle):\n",
    "        \n",
    "        if self.state is not None:\n",
    "            # observe R, next_state and next_action\n",
    "            reward = self.compute_reward(battle)\n",
    "            next_state = self.embed_battle(battle)\n",
    "            next_action = self.choose_action(next_state)\n",
    "            \n",
    "            #alpha\n",
    "            self.N[self.state][self.action] += 1\n",
    "            alpha = 1.0 / self.N[self.state][self.action]\n",
    "            \n",
    "            ########################  Calculate Lambda SARSA\n",
    "            delta = reward + self.gamma*self.Q[next_state][next_action] - self.Q[self.state][self.action]\n",
    "    \n",
    "            self.E[self.state][self.action] += 1\n",
    "    \n",
    "            for s, _ in self.Q.items():\n",
    "                    self.Q[s][:] += alpha * delta * self.E[s][:]\n",
    "                    self.E[s][:] *= self.lambda_ * self.gamma\n",
    "            \n",
    "            # S <- S'  A <- A' \n",
    "            self.state = next_state\n",
    "            self.action = next_action\n",
    "            # Update the policy\n",
    "            self.policy = self.update_epsilon_greedy_policy(self.Q, self.n0, self.N)\n",
    "            \n",
    "        else:\n",
    "            # S first initialization\n",
    "            self.state = self.embed_battle(battle)\n",
    "            # Choose action\n",
    "            self.action = self.choose_action(self.state)\n",
    "\n",
    "        # if the selected action is not possible, perform a random move instead\n",
    "        if self.action == -1:\n",
    "            return ForfeitBattleOrder()\n",
    "        elif self.action < 4 and self.action < len(battle.available_moves) and not battle.force_switch:\n",
    "            return self.create_order(battle.available_moves[self.action])\n",
    "        elif 0 <= self.action - 4 < len(battle.available_switches):\n",
    "            return self.create_order(battle.available_switches[self.action - 4])\n",
    "        else:\n",
    "            return self.choose_random_move(battle)\n",
    "\n",
    "    def _battle_finished_callback(self, battle):\n",
    "        self.E = defaultdict(lambda: np.zeros(N_OUR_ACTIONS))\n",
    "        if use_neptune:\n",
    "            run[f'N0: {self.n0} gamma: {self.gamma} win_acc'].log(self.n_won_battles / len(self._reward_buffer))\n",
    "\n",
    "    \n",
    "    ''' Helper functions '''\n",
    "    #Function to choose the next action\n",
    "    def choose_action(self, state):\n",
    "        action_probs = self.policy(state)\n",
    "        action = np.random.choice(ALL_OUR_ACTIONS, p=action_probs)\n",
    "        #Return the action\n",
    "        return int(action)\n",
    "\n",
    "\n",
    "    # epsilon-greedy policy\n",
    "    def pi(self, state):\n",
    "        epsilon = self.n0 / (self.n0 + np.sum(self.N[state]))\n",
    "        # let's get the greedy action. Ties must be broken arbitrarily\n",
    "        greedy_action = np.random.choice(np.where(self.Q[state] == self.Q[state].max())[0])\n",
    "        action_pick_probability = np.full(N_OUR_ACTIONS, epsilon / N_OUR_ACTIONS)\n",
    "        action_pick_probability[greedy_action] += 1 - epsilon\n",
    "        return np.random.choice(ALL_OUR_ACTIONS, p=action_pick_probability)\n",
    "\n",
    "    # the embed battle is our state\n",
    "    # 12 factors: our active mon, opponent's active mon, 4 moves base power, 4 moves multipliers, num fainted mons\n",
    "    @staticmethod\n",
    "    def embed_battle(battle):\n",
    "        # -1 indicates that the move does not have a base power\n",
    "        # or is not available\n",
    "        moves_base_power = -np.ones(4)\n",
    "        moves_dmg_multiplier = np.ones(4)\n",
    "        for i, move in enumerate(battle.available_moves):\n",
    "            moves_base_power[i] = (\n",
    "                    move.base_power / 100\n",
    "            )  # Simple rescaling to facilitate learning\n",
    "            if move.type:\n",
    "                moves_dmg_multiplier[i] = move.type.damage_multiplier(\n",
    "                    battle.opponent_active_pokemon.type_1,\n",
    "                    battle.opponent_active_pokemon.type_2,\n",
    "                )\n",
    "\n",
    "        # We count how many pokemons have not fainted in each team\n",
    "        n_fainted_mon_team = (\n",
    "            len([mon for mon in battle.team.values() if mon.fainted])\n",
    "        )\n",
    "        n_fainted_mon_opponent = (\n",
    "            len([mon for mon in battle.opponent_team.values() if mon.fainted])\n",
    "        )\n",
    "\n",
    "        state = list()\n",
    "        state.append(NAME_TO_ID_DICT[str(battle.active_pokemon).split(' ')[0]])\n",
    "        state.append(NAME_TO_ID_DICT[str(battle.opponent_active_pokemon).split(' ')[0]])\n",
    "        for move_base_power in moves_base_power:\n",
    "            state.append('{0:.2f}'.format(move_base_power))\n",
    "        for move_dmg_multiplier in moves_dmg_multiplier:\n",
    "            state.append('{0:.2f}'.format(move_dmg_multiplier))\n",
    "        state.append(n_fainted_mon_team)\n",
    "        state.append(n_fainted_mon_opponent)\n",
    "\n",
    "        return str(state)\n",
    "\n",
    "    # Computing rewards\n",
    "    def reward_computing_helper(\n",
    "            self,\n",
    "            battle: AbstractBattle,\n",
    "            *,\n",
    "            fainted_value: float = 0.15,\n",
    "            hp_value: float = 0.15,\n",
    "            number_of_pokemons: int = 6,\n",
    "            starting_value: float = 0.0,\n",
    "            status_value: float = 0.15,\n",
    "            victory_value: float = 1.0\n",
    "    ) -> float:\n",
    "        # 1st compute\n",
    "        if battle not in self._reward_buffer:\n",
    "            self._reward_buffer[battle] = starting_value\n",
    "        current_value = 0\n",
    "\n",
    "        # Verify if pokemon have fainted or have status\n",
    "        for mon in battle.team.values():\n",
    "            current_value += mon.current_hp_fraction * hp_value\n",
    "            if mon.fainted:\n",
    "                current_value -= fainted_value\n",
    "            elif mon.status is not None:\n",
    "                current_value -= status_value\n",
    "\n",
    "        current_value += (number_of_pokemons - len(battle.team)) * hp_value\n",
    "\n",
    "        # Verify if opponent pokemon have fainted or have status\n",
    "        for mon in battle.opponent_team.values():\n",
    "            current_value -= mon.current_hp_fraction * hp_value\n",
    "            if mon.fainted:\n",
    "                current_value += fainted_value\n",
    "            elif mon.status is not None:\n",
    "                current_value += status_value\n",
    "\n",
    "        current_value -= (number_of_pokemons - len(battle.opponent_team)) * hp_value\n",
    "\n",
    "        # Verify if we won or lost\n",
    "        if battle.won:\n",
    "            current_value += victory_value\n",
    "        elif battle.lost:\n",
    "            current_value -= victory_value\n",
    "\n",
    "        # Value to return\n",
    "        to_return = current_value - self._reward_buffer[battle]\n",
    "        self._reward_buffer[battle] = current_value\n",
    "        if use_neptune:\n",
    "            run[f'N0: {self.n0}, gamma: {self.gamma} reward_buffer'].log(current_value)\n",
    "        return to_return\n",
    "\n",
    "    # Calling reward_computing_helper\n",
    "    def compute_reward(self, battle) -> float:\n",
    "        return self.reward_computing_helper(battle, fainted_value=2, hp_value=1, victory_value=15)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Q-learning validation player\n",
    "class SARSAValidationPlayer(PlayerSarsa):\n",
    "    def __init__(self, battle_format, team, Q):\n",
    "        super().__init__(battle_format=battle_format, team=team)\n",
    "        self.Q = Q\n",
    "\n",
    "    def choose_move(self, battle):\n",
    "        state = self.embed_battle(battle)\n",
    "        # let's get the greedy action. Ties must be broken arbitrarily\n",
    "        if state in self.Q.keys():\n",
    "            action = np.random.choice(np.where(self.Q[state] == self.Q[state].max())[0])\n",
    "        else:\n",
    "            return self.choose_random_move(battle)\n",
    "\n",
    "        # if the selected action is not possible, perform a random move instead\n",
    "        if action == -1:\n",
    "            return ForfeitBattleOrder()\n",
    "        elif action < 4 and action < len(battle.available_moves) and not battle.force_switch:\n",
    "            return self.create_order(battle.available_moves[action])\n",
    "        elif 0 <= action - 4 < len(battle.available_switches):\n",
    "            return self.create_order(battle.available_switches[action - 4])\n",
    "        else:\n",
    "            return self.choose_random_move(battle)\n",
    "\n",
    "    def _battle_finished_callback(self, battle):\n",
    "        pass\n",
    "\n",
    "    # the embed battle is our state\n",
    "    # 12 factors: our active mon, opponent's active mon, 4 moves base power, 4 moves multipliers, remaining mons\n",
    "    @staticmethod\n",
    "    def embed_battle(battle):\n",
    "        # -1 indicates that the move does not have a base power\n",
    "        # or is not available\n",
    "        moves_base_power = -np.ones(4)\n",
    "        moves_dmg_multiplier = np.ones(4)\n",
    "        for i, move in enumerate(battle.available_moves):\n",
    "            moves_base_power[i] = (\n",
    "                    move.base_power / 100\n",
    "            )  # Simple rescaling to facilitate learning\n",
    "            if move.type:\n",
    "                moves_dmg_multiplier[i] = move.type.damage_multiplier(\n",
    "                    battle.opponent_active_pokemon.type_1,\n",
    "                    battle.opponent_active_pokemon.type_2,\n",
    "                )\n",
    "\n",
    "        # We count how many pokemons have not fainted in each team\n",
    "        remaining_mon_team = (\n",
    "            len([mon for mon in battle.team.values() if mon.fainted])\n",
    "        )\n",
    "        remaining_mon_opponent = (\n",
    "            len([mon for mon in battle.opponent_team.values() if mon.fainted])\n",
    "        )\n",
    "\n",
    "        state = list()\n",
    "        state.append(NAME_TO_ID_DICT[str(battle.active_pokemon).split(' ')[0]])\n",
    "        state.append(NAME_TO_ID_DICT[str(battle.opponent_active_pokemon).split(' ')[0]])\n",
    "        for move_base_power in moves_base_power:\n",
    "            state.append('{0:.2f}'.format(move_base_power))\n",
    "        for move_dmg_multiplier in moves_dmg_multiplier:\n",
    "            state.append('{0:.2f}'.format(move_dmg_multiplier))\n",
    "        state.append(remaining_mon_team)\n",
    "        state.append(remaining_mon_opponent)\n",
    "\n",
    "        return str(state)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# global parameters\n",
    "\n",
    "# possible values for num_battles (number of episodes)\n",
    "n_battles_array = [10000]\n",
    "# exploration schedule from MC, i. e., epsilon(t) = N0 / (N0 + N(S(t)))\n",
    "n0_array = [0.0001, 0.001, 0.01]\n",
    "# possible values for gamma (discount factor)\n",
    "gamma_array = [0.75] \n",
    "#Lambda\n",
    "lambda_ = [0, 0.2, 0.4, 0.6, 0.8, 1]\n",
    "\n",
    "list_of_params = [\n",
    "    {\n",
    "        'n_battles': n_battles,\n",
    "        'n0': n0,\n",
    "        'gamma': gamma,\n",
    "        'lambda': lambda_\n",
    "    } for n_battles, n0, gamma, lambda_ in product(n_battles_array, n0_array, gamma_array, lambda_)\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# json helper functions\n",
    "\n",
    "def save_dict_to_json(path_dir, filename, data, append=True):\n",
    "    if not os.path.exists(path_dir):\n",
    "        os.makedirs(path_dir)\n",
    "    full_filename = path_dir + \"/\" + filename\n",
    "    if os.path.exists(full_filename) and append:\n",
    "        with open(full_filename, \"r\") as file:\n",
    "            value_dict = json.load(file)\n",
    "            for key in data:\n",
    "                value_dict[key] = data[key] if isinstance(data[key], list) else data[key].tolist()\n",
    "            file.close()\n",
    "    else:\n",
    "        value_dict = dict()\n",
    "        for key in data:\n",
    "            value_dict[key] = data[key] if isinstance(data[key], list) else data[key].tolist()\n",
    "    # write\n",
    "    with open(full_filename, \"w\") as file:\n",
    "        json.dump(value_dict, file)\n",
    "        file.close()\n",
    "\n",
    "\n",
    "def read_dict_from_json(path_dir, filename):\n",
    "    full_filename = path_dir + \"/\" + filename\n",
    "    if not os.path.exists(full_filename):\n",
    "        return dict()\n",
    "    file = open(full_filename, \"r\")\n",
    "    data = json.load(file)\n",
    "    file.close()\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let's battle!\n",
    "async def lets_battle():\n",
    "    for params in list_of_params:\n",
    "        start = time.time()\n",
    "        if use_neptune:\n",
    "            run['params'] = params\n",
    "        params['player'] = SARSAPlayer(battle_format=\"gen8ou\", team=OUR_TEAM, n0=params['n0'], gamma=params['gamma'], lambda_ = params['lambda'])\n",
    "        params['opponent'] = MaxDamagePlayer(battle_format=\"gen8ou\", team=OP_TEAM)\n",
    "        await params['player'].battle_against(opponent=params['opponent'], n_battles=params['n_battles'])\n",
    "        if debug:\n",
    "            print(\"training: num battles (episodes)=%d, N0=%f, gamma=%f, lambda=%f, wins=%d, winning %%=%f, total time=%s sec\" %\n",
    "                  (\n",
    "                      params['n_battles'],\n",
    "                      round(params['n0'], 2),\n",
    "                      round(params['gamma'], 2),\n",
    "                      round(params['lambda'], 2),\n",
    "                      params['player'].n_won_battles,\n",
    "                      round((params['player'].n_won_battles / params['n_battles']) * 100, 2),\n",
    "                      round(time.time() - start, 2)\n",
    "                  ))\n",
    "\n",
    "        # save Q to json file\n",
    "        if save_to_json_file:\n",
    "            today_s = str(date.today())\n",
    "            n_battle_s = str(params['n_battles'])\n",
    "            n0_s = str(round(params['n0'], 8))\n",
    "            gamma_s = str(round(params['gamma'], 8))\n",
    "            lambda_s = str(round(params['lambda'], 8))\n",
    "            winning_percentage_s = str(round((params['player'].n_won_battles / params['n_battles']) * 100, 2))\n",
    "            filename = \"SarsaLambda_\" + today_s + \"_\" + n_battle_s + \"_\" + n0_s + \"_\" + gamma_s + \"_\" + lambda_s + \"_\" + winning_percentage_s + \".json\"\n",
    "            save_dict_to_json(\"./SarsaLambda10k_Table\", filename, params['player'].Q, False)\n",
    "\n",
    "        # statistics: key: \"n_battles, n0, gamma\", values: list of win or lose\n",
    "        key = str(params['n_battles']) + \"_\" + str(round(params['n0'], 2)) + \"_\" + str(round(params['gamma'], 2)) + \"_\" + str(round(params['lambda'], 2))\n",
    "        winning_status = list()\n",
    "        for battle in params['player']._battles.values():\n",
    "            if battle.won:\n",
    "                winning_status.append(True)\n",
    "            else:\n",
    "                winning_status.append(False)\n",
    "        # save statistics json file (append)\n",
    "        data = dict()\n",
    "        data[key] = winning_status\n",
    "        save_dict_to_json(\"./Sarsa10k_statistics\", \"statistics.json\", data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.000000, gamma=0.750000, lambda=0.000000, wins=5362, winning %=53.620000, total time=5226.77 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-03 14:20:18,951 - MaxDamagePlayer 1 - WARNING - Trying to login as MaxDamagePlayer 1, showdown returned  MaxDamagePlayer 1@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 14:20:18,953 - SARSAPlayer 1 - WARNING - Trying to login as SARSAPlayer 1, showdown returned  SARSAPlayer 1@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.000000, gamma=0.750000, lambda=0.200000, wins=4785, winning %=47.850000, total time=5086.04 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-03 15:50:18,964 - MaxDamagePlayer 2 - WARNING - Trying to login as MaxDamagePlayer 2, showdown returned  MaxDamagePlayer 2@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 15:50:18,965 - SARSAPlayer 2 - WARNING - Trying to login as SARSAPlayer 2, showdown returned  SARSAPlayer 2@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.000000, gamma=0.750000, lambda=0.400000, wins=4645, winning %=46.450000, total time=4737.21 sec\n",
      "training: num battles (episodes)=10000, N0=0.000000, gamma=0.750000, lambda=0.600000, wins=2579, winning %=25.790000, total time=4497.92 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-03 17:20:18,973 - SARSAPlayer 3 - WARNING - Trying to login as SARSAPlayer 3, showdown returned  SARSAPlayer 3@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 17:20:18,975 - MaxDamagePlayer 3 - WARNING - Trying to login as MaxDamagePlayer 3, showdown returned  MaxDamagePlayer 3@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.000000, gamma=0.750000, lambda=0.800000, wins=1525, winning %=15.250000, total time=4386.0 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-03 18:20:18,976 - SARSAPlayer 4 - WARNING - Trying to login as SARSAPlayer 4, showdown returned  SARSAPlayer 4@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 18:20:18,977 - MaxDamagePlayer 4 - WARNING - Trying to login as MaxDamagePlayer 4, showdown returned  MaxDamagePlayer 4@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 19:20:18,979 - SARSAPlayer 5 - WARNING - Trying to login as SARSAPlayer 5, showdown returned  SARSAPlayer 5@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 19:20:18,980 - MaxDamagePlayer 5 - WARNING - Trying to login as MaxDamagePlayer 5, showdown returned  MaxDamagePlayer 5@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.000000, gamma=0.750000, lambda=1.000000, wins=887, winning %=8.870000, total time=4022.22 sec\n",
      "training: num battles (episodes)=10000, N0=0.000000, gamma=0.750000, lambda=0.000000, wins=5102, winning %=51.020000, total time=4987.39 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-03 20:50:18,993 - SARSAPlayer 6 - WARNING - Trying to login as SARSAPlayer 6, showdown returned  SARSAPlayer 6@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 20:50:18,993 - MaxDamagePlayer 6 - WARNING - Trying to login as MaxDamagePlayer 6, showdown returned  MaxDamagePlayer 6@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 21:50:18,995 - MaxDamagePlayer 7 - WARNING - Trying to login as MaxDamagePlayer 7, showdown returned  MaxDamagePlayer 7@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 21:50:18,996 - SARSAPlayer 7 - WARNING - Trying to login as SARSAPlayer 7, showdown returned  SARSAPlayer 7@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.000000, gamma=0.750000, lambda=0.200000, wins=4635, winning %=46.350000, total time=5014.26 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-03 23:20:19,005 - SARSAPlayer 8 - WARNING - Trying to login as SARSAPlayer 8, showdown returned  SARSAPlayer 8@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 23:20:19,006 - MaxDamagePlayer 8 - WARNING - Trying to login as MaxDamagePlayer 8, showdown returned  MaxDamagePlayer 8@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.000000, gamma=0.750000, lambda=0.400000, wins=3909, winning %=39.090000, total time=4488.33 sec\n",
      "training: num battles (episodes)=10000, N0=0.000000, gamma=0.750000, lambda=0.600000, wins=2550, winning %=25.500000, total time=4509.55 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-04 00:50:19,009 - SARSAPlayer 9 - WARNING - Trying to login as SARSAPlayer 9, showdown returned  SARSAPlayer 9@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 00:50:19,010 - MaxDamagePlayer 9 - WARNING - Trying to login as MaxDamagePlayer 9, showdown returned  MaxDamagePlayer 9@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 01:50:19,021 - SARSAPlayer 10 - WARNING - Trying to login as SARSAPlayer 10, showdown returned  SARSAPlayer 10@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 01:50:19,022 - MaxDamagePlayer 10 - WARNING - Trying to login as MaxDamagePlayer 10, showdown returned  MaxDamagePlayer 10@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.000000, gamma=0.750000, lambda=0.800000, wins=1505, winning %=15.050000, total time=4303.76 sec\n",
      "training: num battles (episodes)=10000, N0=0.000000, gamma=0.750000, lambda=1.000000, wins=814, winning %=8.140000, total time=4084.27 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-04 03:20:19,029 - SARSAPlayer 11 - WARNING - Trying to login as SARSAPlayer 11, showdown returned  SARSAPlayer 11@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 03:20:19,030 - MaxDamagePlayer 11 - WARNING - Trying to login as MaxDamagePlayer 11, showdown returned  MaxDamagePlayer 11@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 04:20:19,040 - SARSAPlayer 12 - WARNING - Trying to login as SARSAPlayer 12, showdown returned  SARSAPlayer 12@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 04:20:19,041 - MaxDamagePlayer 12 - WARNING - Trying to login as MaxDamagePlayer 12, showdown returned  MaxDamagePlayer 12@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.010000, gamma=0.750000, lambda=0.000000, wins=4275, winning %=42.750000, total time=5235.73 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-04 05:50:19,041 - SARSAPlayer 13 - WARNING - Trying to login as SARSAPlayer 13, showdown returned  SARSAPlayer 13@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 05:50:19,051 - MaxDamagePlayer 13 - WARNING - Trying to login as MaxDamagePlayer 13, showdown returned  MaxDamagePlayer 13@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.010000, gamma=0.750000, lambda=0.200000, wins=4665, winning %=46.650000, total time=4927.17 sec\n",
      "training: num battles (episodes)=10000, N0=0.010000, gamma=0.750000, lambda=0.400000, wins=4139, winning %=41.390000, total time=4960.73 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-04 07:20:19,055 - SARSAPlayer 14 - WARNING - Trying to login as SARSAPlayer 14, showdown returned  SARSAPlayer 14@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 07:20:19,056 - MaxDamagePlayer 14 - WARNING - Trying to login as MaxDamagePlayer 14, showdown returned  MaxDamagePlayer 14@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 08:20:19,061 - SARSAPlayer 15 - WARNING - Trying to login as SARSAPlayer 15, showdown returned  SARSAPlayer 15@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 08:20:19,062 - MaxDamagePlayer 15 - WARNING - Trying to login as MaxDamagePlayer 15, showdown returned  MaxDamagePlayer 15@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.010000, gamma=0.750000, lambda=0.600000, wins=2131, winning %=21.310000, total time=4561.34 sec\n",
      "training: num battles (episodes)=10000, N0=0.010000, gamma=0.750000, lambda=0.800000, wins=1394, winning %=13.940000, total time=4590.49 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-04 09:50:19,076 - MaxDamagePlayer 16 - WARNING - Trying to login as MaxDamagePlayer 16, showdown returned  MaxDamagePlayer 16@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 09:50:19,092 - SARSAPlayer 16 - WARNING - Trying to login as SARSAPlayer 16, showdown returned  SARSAPlayer 16@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 10:50:19,071 - SARSAPlayer 17 - WARNING - Trying to login as SARSAPlayer 17, showdown returned  SARSAPlayer 17@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-04 10:50:19,073 - MaxDamagePlayer 17 - WARNING - Trying to login as MaxDamagePlayer 17, showdown returned  MaxDamagePlayer 17@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.010000, gamma=0.750000, lambda=1.000000, wins=623, winning %=6.230000, total time=4219.74 sec\n"
     ]
    }
   ],
   "source": [
    "loop = asyncio.get_event_loop()\n",
    "loop.run_until_complete(loop.create_task(lets_battle()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plotting helper functions\n",
    "\n",
    "def plot_2d(path, title, x_label, x_array, y_label, y_array):\n",
    "    # set labels and plot surface\n",
    "    figure = matplotlib.pyplot.figure(figsize=(20, 10))\n",
    "    ax = figure.gca()\n",
    "    ax.set_title(title)\n",
    "    ax.set_xlabel(x_label)\n",
    "    ax.set_ylabel(y_label)\n",
    "    ax.plot(x_array, y_array)\n",
    "    # pyplot.show()\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "    filename = path + \"/\" + title + \"_\" + x_label + \"_\" + y_label + \"_\" + \".pdf\"\n",
    "    figure.savefig(filename, dpi=figure.dpi)\n",
    "    pyplot.close(figure)\n",
    "\n",
    "def plot_3d(path, title, x_label, x_array, y_label, y_array, z_label, z_array):\n",
    "    xyz = {'x': x_array, 'y': y_array, 'z': z_array}\n",
    "    df = pd.DataFrame(xyz, index=range(len(xyz['x'])))\n",
    "    xv, yv = np.meshgrid(x_array, y_array)\n",
    "    zv = griddata((df['x'], df['y']), df['z'], (xv, yv), method='nearest')\n",
    "    # set labels and plot surface\n",
    "    figure = matplotlib.pyplot.figure(figsize=(20, 10))\n",
    "    ax = figure.gca(projection='3d')\n",
    "    ax.set_title(title)\n",
    "    ax.set_xlabel(x_label)\n",
    "    ax.set_ylabel(y_label)\n",
    "    ax.set_zlabel(z_label)\n",
    "    surface = ax.plot_surface(xv, yv, zv, rstride=1, cstride=1, cmap=matplotlib.cm.coolwarm, linewidth=0,\n",
    "                              antialiased=False)\n",
    "    figure.colorbar(surface)\n",
    "    # pyplot.show()\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "    filename = path + \"/\" + title + \".pdf\"\n",
    "    figure.savefig(filename, dpi=figure.dpi)\n",
    "    pyplot.close(figure)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot value from state-action pair\n",
    "\n",
    "def plot_v_from_state_action_json(path_dir, x_label, x_func, y_label, y_func):\n",
    "    # open files\n",
    "    for filename in os.listdir(path_dir):\n",
    "        q = dict()\n",
    "        q_json = read_dict_from_json(path_dir, filename)\n",
    "        for key in q_json.keys():\n",
    "            q[key] = np.array(q_json[key])\n",
    "        x_values = []\n",
    "        y_values = []\n",
    "        z_values = []\n",
    "        # for state, actions\n",
    "        for state, actions in q.items():\n",
    "            state = re.sub(r\"[,!?><:'\\[\\]()@*~#]\", \"\", state)\n",
    "            key_float = [float(k) for k in state.split()]\n",
    "            x_emb = x_func(key_float)\n",
    "            x_values.append(x_emb)\n",
    "            y_emb = y_func(key_float)\n",
    "            y_values.append(y_emb)\n",
    "            action_value = np.max(actions)\n",
    "            z_values.append(action_value)\n",
    "        # plot 3D\n",
    "        title = \"v_from_\" + filename\n",
    "        plot_3d(\"./plot\", title, x_label, x_values, y_label, y_values, \"V\", z_values)\n",
    "\n",
    "\n",
    "# plots from Q\n",
    "plot_v_from_state_action_json(path_dir=\"./best_SarsaLambda10k_Table\",\n",
    "                              x_label=\"20 * index_pokemon + sum(moves_base_power * moves_dmg_multiplier)\",\n",
    "                              x_func=lambda k: 20 * k[0] + k[1] * k[5] + k[2] * k[6] + k[3] * k[7] + k[4] * k[8],\n",
    "                              y_label=\"remaining_mon_team - remaining_mon_opponent\",\n",
    "                              y_func=lambda k: k[8] - k[9])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot additional statistics\n",
    "\n",
    "def plot_statistics_json(path_dir, filename=\"statistics.json\"):\n",
    "    statistics = read_dict_from_json(path_dir, filename)\n",
    "    # win/lost vs. episode number\n",
    "    for key in statistics.keys():\n",
    "        key_elements = key.split(\"_\")\n",
    "        n_battles = key_elements[0]\n",
    "        n0 = key_elements[1]\n",
    "        gamma = key_elements[2]\n",
    "        value = statistics[key]\n",
    "        plot_2d(path=\"./plot\",\n",
    "                title=\"victory_n_battles_\" + n_battles + \"_N0_\" + n0 + \"_gamma_\" + gamma,\n",
    "                x_label=\"episodes\",\n",
    "                x_array=np.array(range(0, len(value))),\n",
    "                y_label=\"victory\",\n",
    "                y_array=np.array(value).astype(int))\n",
    "\n",
    "    # winning % by set of parameters\n",
    "    n_battles = \"\"\n",
    "    x_values = []\n",
    "    y_values = []\n",
    "    z_values = []\n",
    "    for key in statistics.keys():\n",
    "        key_elements = key.split(\"_\")\n",
    "        n_battles = key_elements[0]\n",
    "        n0 = key_elements[1]\n",
    "        gamma = key_elements[2]\n",
    "        value = statistics[key]\n",
    "        x_values.append(n0)\n",
    "        y_values.append(gamma)\n",
    "        z_values.append(value.count(True) / len(value))\n",
    "    plot_3d(path=\"./plot\",\n",
    "            title=\"winning_percentage_n_battles_\" + n_battles,\n",
    "            x_label=\"N0\",\n",
    "            x_array=np.array(x_values).astype(np.float),\n",
    "            y_label=\"gamma\",\n",
    "            y_array=np.array(y_values).astype(np.float),\n",
    "            z_label=\"winning %\",\n",
    "            z_array=np.array(z_values))\n",
    "\n",
    "\n",
    "# plots from statistics\n",
    "plot_statistics_json(\"./statistics\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# validation\n",
    "\n",
    "async def do_battle_validation():\n",
    "    for params in list_of_params:\n",
    "        # validation (play 1/3 of the battles using Q-learned table)\n",
    "        start = time.time()\n",
    "        params['player'] = SARSAPlayer(battle_format=\"gen8ou\", team=OUR_TEAM, n0=params['n0'], gamma=params['gamma'], lambda_ = params['lambda'])\n",
    "        params['opponent'] = MaxDamagePlayer(battle_format=\"gen8ou\", team=OP_TEAM)\n",
    "        params['validation_player'] = SARSAValidationPlayer(battle_format=\"gen8ou\", team=OUR_TEAM, Q=params['player'].Q)\n",
    "        n_battles = int(params['n_battles'] / 5)\n",
    "        await params['validation_player'].battle_against(opponent=params['opponent'], n_battles=n_battles)\n",
    "        print(\"training: num battles (episodes)=%d, N0=%f, gamma=%f, lambda=%f, wins=%d, winning %%=%f, total time=%s sec\" %\n",
    "                  (\n",
    "                      params['n_battles'],\n",
    "                      round(params['n0'], 5),\n",
    "                      round(params['gamma'], 2),\n",
    "                      round(params['lambda'], 2),\n",
    "                      params['player'].n_won_battles,\n",
    "                      round((params['player'].n_won_battles / params['n_battles']) * 100, 2),\n",
    "                      round(time.time() - start, 2)\n",
    "                  ))\n",
    "\n",
    "loop = asyncio.get_event_loop()\n",
    "loop.run_until_complete(loop.create_task(do_battle_validation()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for filename in os.listdir(path_dir):\n",
    "    Q = np.array(read_array_from_json(\"./SarsaLambda10k_Table_best\", filename))\n",
    "    print(filename)\n",
    "    print(Q)\n",
    "    break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}