MC_Control_FA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bd611aca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "\n",
    "import asyncio\n",
    "import json\n",
    "import os\n",
    "import matplotlib\n",
    "import neptune.new as neptune\n",
    "import nest_asyncio\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "\n",
    "from collections import defaultdict\n",
    "from datetime import date\n",
    "from itertools import product\n",
    "from matplotlib import pyplot\n",
    "from poke_env.environment.abstract_battle import AbstractBattle\n",
    "from poke_env.player.battle_order import ForfeitBattleOrder\n",
    "from poke_env.player.player import Player\n",
    "# from poke_env.player.random_player import RandomPlayer\n",
    "from scipy.interpolate import griddata\n",
    "from src.playerMC_FA import Player as PlayerMC_FA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1f537cc4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://app.neptune.ai/leolellisr/rl-pokeenv/e/RLPOK-94\n",
      "Remember to stop your run once you’ve finished logging your metadata (https://docs.neptune.ai/api-reference/run#stop). It will be stopped automatically only when the notebook kernel/interactive console is terminated.\n"
     ]
    }
   ],
   "source": [
    "# global configs\n",
    "\n",
    "debug = True\n",
    "save_to_json_file = True\n",
    "use_validation = True\n",
    "use_neptune = True\n",
    "\n",
    "nest_asyncio.apply()\n",
    "np.random.seed(0)\n",
    "\n",
    "if use_neptune:\n",
    "    run = neptune.init(project='leolellisr/rl-pokeenv',\n",
    "                       api_token='eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLm5lcHR1bmUuYWkiLCJhcGlfdXJsIjoiaHR0cHM6Ly9hcHAubmVwdHVuZS5haSIsImFwaV9rZXkiOiI1NjY1YmJkZi1hYmM5LTQ3M2QtOGU1ZC1iZTFlNWY4NjE1NDQifQ==')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "093e462f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# our team\n",
    "\n",
    "OUR_TEAM = \"\"\"\n",
    "Pikachu-Original (M) @ Light Ball  \n",
    "Ability: Static  \n",
    "EVs: 252 Atk / 4 SpD / 252 Spe  \n",
    "Jolly Nature  \n",
    "- Volt Tackle  \n",
    "- Nuzzle  \n",
    "- Iron Tail  \n",
    "- Knock Off  \n",
    "\n",
    "Charizard @ Life Orb  \n",
    "Ability: Solar Power  \n",
    "EVs: 252 SpA / 4 SpD / 252 Spe  \n",
    "Timid Nature  \n",
    "IVs: 0 Atk  \n",
    "- Flamethrower  \n",
    "- Dragon Pulse  \n",
    "- Roost  \n",
    "- Sunny Day  \n",
    "\n",
    "Blastoise @ White Herb  \n",
    "Ability: Torrent  \n",
    "EVs: 4 Atk / 252 SpA / 252 Spe  \n",
    "Mild Nature  \n",
    "- Scald  \n",
    "- Ice Beam  \n",
    "- Earthquake  \n",
    "- Shell Smash  \n",
    "\n",
    "Venusaur @ Black Sludge  \n",
    "Ability: Chlorophyll  \n",
    "EVs: 252 SpA / 4 SpD / 252 Spe  \n",
    "Modest Nature  \n",
    "IVs: 0 Atk  \n",
    "- Giga Drain  \n",
    "- Sludge Bomb  \n",
    "- Sleep Powder  \n",
    "- Leech Seed  \n",
    "\n",
    "Sirfetch’d @ Aguav Berry  \n",
    "Ability: Steadfast  \n",
    "EVs: 248 HP / 252 Atk / 8 SpD  \n",
    "Adamant Nature  \n",
    "- Close Combat  \n",
    "- Swords Dance  \n",
    "- Poison Jab  \n",
    "- Knock Off  \n",
    "\n",
    "Tauros (M) @ Assault Vest  \n",
    "Ability: Intimidate  \n",
    "EVs: 252 Atk / 4 SpD / 252 Spe  \n",
    "Jolly Nature  \n",
    "- Double-Edge  \n",
    "- Earthquake  \n",
    "- Megahorn  \n",
    "- Iron Head  \n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "12635b18",
   "metadata": {},
   "outputs": [],
   "source": [
    "# opponent's team\n",
    "\n",
    "OP_TEAM = \"\"\"\n",
    "Eevee @ Eviolite  \n",
    "Ability: Adaptability  \n",
    "EVs: 252 HP / 252 Atk / 4 SpD  \n",
    "Adamant Nature  \n",
    "- Quick Attack  \n",
    "- Flail  \n",
    "- Facade  \n",
    "- Wish  \n",
    "\n",
    "Vaporeon @ Leftovers  \n",
    "Ability: Hydration  \n",
    "EVs: 252 HP / 252 Def / 4 SpA  \n",
    "Bold Nature  \n",
    "IVs: 0 Atk  \n",
    "- Scald  \n",
    "- Shadow Ball  \n",
    "- Toxic  \n",
    "- Wish  \n",
    "\n",
    "Sylveon @ Aguav Berry  \n",
    "Ability: Pixilate  \n",
    "EVs: 252 HP / 252 SpA / 4 SpD  \n",
    "Modest Nature  \n",
    "IVs: 0 Atk  \n",
    "- Hyper Voice  \n",
    "- Mystical Fire  \n",
    "- Psyshock  \n",
    "- Calm Mind  \n",
    "\n",
    "Jolteon @ Assault Vest  \n",
    "Ability: Quick Feet  \n",
    "EVs: 252 SpA / 4 SpD / 252 Spe  \n",
    "Timid Nature  \n",
    "IVs: 0 Atk  \n",
    "- Thunderbolt  \n",
    "- Hyper Voice  \n",
    "- Volt Switch  \n",
    "- Shadow Ball  \n",
    "\n",
    "Leafeon @ Life Orb  \n",
    "Ability: Chlorophyll  \n",
    "EVs: 252 Atk / 4 SpD / 252 Spe  \n",
    "Adamant Nature  \n",
    "- Leaf Blade  \n",
    "- Knock Off  \n",
    "- X-Scissor  \n",
    "- Swords Dance  \n",
    "\n",
    "Umbreon @ Iapapa Berry  \n",
    "Ability: Inner Focus  \n",
    "EVs: 252 HP / 4 Atk / 252 SpD  \n",
    "Careful Nature  \n",
    "- Foul Play  \n",
    "- Body Slam  \n",
    "- Toxic  \n",
    "- Wish  \n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f27f14a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "N_STATE_COMPONENTS = 12\n",
    "# num of features = num of state components + action\n",
    "N_FEATURES = N_STATE_COMPONENTS + 1\n",
    "\n",
    "N_OUR_MOVE_ACTIONS = 4\n",
    "N_OUR_SWITCH_ACTIONS = 5\n",
    "N_OUR_ACTIONS = N_OUR_MOVE_ACTIONS + N_OUR_SWITCH_ACTIONS\n",
    "\n",
    "ALL_OUR_ACTIONS = np.array(range(0, N_OUR_ACTIONS))\n",
    "\n",
    "def name_to_id(name):\n",
    "    if(name == 'venusaur'): return 0\n",
    "    if(name == 'pikachuoriginal'): return 1 \n",
    "    if(name == 'tauros'): return 2 \n",
    "    if(name == 'sirfetchd'): return 3 \n",
    "    if(name == 'blastoise'): return 4 \n",
    "    if(name == 'charizard'): return 5 \n",
    "    if(name == 'eevee'): return 0\n",
    "    if(name == 'vaporeon'): return 1 \n",
    "    if(name == 'leafeon'): return 2 \n",
    "    if(name == 'sylveon'): return 3 \n",
    "    if(name == 'jolteon'): return 4 \n",
    "    if(name == 'umbreon'): return 5     \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "22dada00",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Max-damage player\n",
    "\n",
    "class MaxDamagePlayer(Player):\n",
    "    def choose_move(self, battle):\n",
    "        if battle.available_moves:\n",
    "            best_move = max(battle.available_moves, key=lambda move: move.base_power)\n",
    "            return self.create_order(best_move)\n",
    "        else:\n",
    "            return self.choose_random_move(battle)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "fc1b7c73",
   "metadata": {},
   "outputs": [],
   "source": [
    "# MC Control FA player\n",
    "\n",
    "class MCPlayer(PlayerMC_FA):\n",
    "    def choose_move(self, battle):\n",
    "        # In the 1st state of all we don't append yet;\n",
    "        # Other states: using previous state and action with actual reward (actual battle) \n",
    "        if self.previous_action != -10: self.episode.append((self.previous_state, self.previous_action, self.compute_reward(battle)))\n",
    "            \n",
    "        # Getting state s (-> embed battle)\n",
    "        s = self.embed_battle(battle)\n",
    "        # 1st move will be random wout policy    \n",
    "        if(self.aux == 0):\n",
    "            self.aux = 1\n",
    "            action = np.random.choice(self.action_space)\n",
    "\n",
    "        # Other moves have policy    \n",
    "        else: action = np.random.choice(self.action_space, p=self.policy(s))\n",
    "\n",
    "        # Saving action and state to append later. We can compute the reward only after the move    \n",
    "        self.previous_action = action\n",
    "        self.previous_state = s\n",
    "\n",
    "        # Choose move according to action index\n",
    "        if action == -1:\n",
    "            return ForfeitBattleOrder()\n",
    "        elif (\n",
    "            action < 4\n",
    "            and action < len(battle.available_moves)\n",
    "            and not battle.force_switch\n",
    "        ):\n",
    "            return self.create_order(battle.available_moves[action])\n",
    "        elif 0 <= action - 4 < len(battle.available_switches):\n",
    "            return self.create_order(battle.available_switches[action - 4])\n",
    "        else:\n",
    "            return self.choose_random_move(battle)\n",
    "        \n",
    "    \n",
    "    def x(self, state, action):\n",
    "        state = np.array(state).astype(float)\n",
    "        return np.append(state, action)\n",
    "\n",
    "    # q^(S, A, W)\n",
    "    def q_approx(self, state, action, w):\n",
    "        state = np.array(state).astype(float)\n",
    "        return np.dot(self.x(state, action), w)\n",
    "\n",
    "    # max(a, q^(S, a', W))\n",
    "    def max_q_approx(self, state, w):\n",
    "        state = np.array(state).astype(float)\n",
    "        return max(np.array([self.q_approx(state, action, w) for action in range(N_OUR_ACTIONS)]))\n",
    "\n",
    "    def _battle_finished_callback(self, battle):\n",
    "        rewards = [reward for state,action,reward in self.episode]\n",
    "        states = [state for state,action,reward in self.episode]\n",
    "        actions = [action for state,action,reward in self.episode]\n",
    "        t_array = range(len(self.episode)+1)\n",
    "        # Computing Q and N\n",
    "        for idx, state in enumerate(states):\n",
    "            action = actions[idx]\n",
    "            if state not in self.visited_states:\n",
    "                self.N[str(state)][action] += 1\n",
    "                returnGt = sum([reward*pow(self.gamma, t) for reward, t in zip(rewards[idx:], t_array[:(-idx-1)])]) \n",
    "\n",
    "                # incremental update of Q value is more efficient than keeping a record of all rewards\n",
    "                # and averaging after every new reward\n",
    "                # step-size: 1./N[state][action]\n",
    "                delta = \\\n",
    "                returnGt - self.q_approx(state, action, self.w)\n",
    "                self.w += (1/self.N[str(state)][action])* delta * self.x(state, action)\n",
    "            \n",
    "                #run[f'N0: {self.n0} gamma: {self.gamma} q_value'].log(self.Q[state][action])\n",
    "                self.visited_states.append(state)\n",
    "                \n",
    "        self.visited_states = []\n",
    "        self.episode = []\n",
    "        if(self.aux == 1):\n",
    "            self.aux == 0\n",
    "            \n",
    "        # Define new policy with updated Q and N\n",
    "        self.policy = self.update_epsilon_greedy_policy(self.w, self.n0, self.N)\n",
    "        if use_neptune: run[f'N0: {self.n0} gamma: {self.gamma} win_acc'].log(self.n_won_battles/len(self._reward_buffer))\n",
    "            \n",
    "    ''' Helper functions '''\n",
    "\n",
    "    # the embed battle is our state\n",
    "    # 10 factors: 4 moves base power, 4 moves multipliers, active pokemon and active opponent pokemon \n",
    "    def embed_battle(self, battle):\n",
    "        # -1 indicates that the move does not have a base power or not available\n",
    "        moves_base_power = -np.ones(4)\n",
    "        moves_dmg_multiplier = np.ones(4)\n",
    "        for i, move in enumerate(battle.available_moves):\n",
    "            moves_base_power[i] = (\n",
    "                move.base_power / 100\n",
    "            )  # Simple rescaling \n",
    "            if move.type:\n",
    "                moves_dmg_multiplier[i] = move.type.damage_multiplier(\n",
    "                    battle.opponent_active_pokemon.type_1,\n",
    "                    battle.opponent_active_pokemon.type_2,\n",
    "                )\n",
    "\n",
    "        # how many pokemons have fainted in each team\n",
    "        remaining_mon_team = (\n",
    "            len([mon for mon in battle.team.values() if mon.fainted]) / 6\n",
    "        )\n",
    "        remaining_mon_opponent = (\n",
    "            len([mon for mon in battle.opponent_team.values() if mon.fainted]) / 6\n",
    "        )\n",
    "\n",
    "        state = list()       \n",
    "        state.append('{0:.2f}'.format(name_to_id(str(battle.active_pokemon).split(' ')[0])))\n",
    "        state.append('{0:.2f}'.format(name_to_id(str(battle.opponent_active_pokemon).split(' ')[0])))    \n",
    "        for move_base_power in moves_base_power:\n",
    "            state.append('{0:.2f}'.format(move_base_power))\n",
    "        for move_dmg_multiplier in moves_dmg_multiplier:\n",
    "            state.append('{0:.2f}'.format(move_dmg_multiplier))\n",
    "        state.append('{0:.2f}'.format(remaining_mon_team))\n",
    "        state.append('{0:.2f}'.format(remaining_mon_opponent))\n",
    "        # Convert to string so we can use as hash\n",
    "        return state\n",
    "\n",
    "    # Computing rewards\n",
    "    def reward_computing_helper(\n",
    "            self,\n",
    "            battle: AbstractBattle,\n",
    "            *,\n",
    "            fainted_value: float = 0.15,\n",
    "            hp_value: float = 0.15,\n",
    "            number_of_pokemons: int = 6,\n",
    "            starting_value: float = 0.0,\n",
    "            status_value: float = 0.15,\n",
    "            victory_value: float = 1.0\n",
    "    ) -> float:\n",
    "        # 1st compute\n",
    "        if battle not in self._reward_buffer:\n",
    "            self._reward_buffer[battle] = starting_value\n",
    "        current_value = 0\n",
    "\n",
    "        # Verify if pokemon have fainted or have status\n",
    "        for mon in battle.team.values():\n",
    "            current_value += mon.current_hp_fraction * hp_value\n",
    "            if mon.fainted:\n",
    "                current_value -= fainted_value\n",
    "            elif mon.status is not None:\n",
    "                current_value -= status_value\n",
    "\n",
    "        current_value += (number_of_pokemons - len(battle.team)) * hp_value\n",
    "\n",
    "        # Verify if opponent pokemon have fainted or have status\n",
    "        for mon in battle.opponent_team.values():\n",
    "            current_value -= mon.current_hp_fraction * hp_value\n",
    "            if mon.fainted:\n",
    "                current_value += fainted_value\n",
    "            elif mon.status is not None:\n",
    "                current_value += status_value\n",
    "\n",
    "        current_value -= (number_of_pokemons - len(battle.opponent_team)) * hp_value\n",
    "\n",
    "        # Verify if we won or lost\n",
    "        if battle.won:\n",
    "            current_value += victory_value\n",
    "        elif battle.lost:\n",
    "            current_value -= victory_value\n",
    "\n",
    "        # Value to return\n",
    "        to_return = current_value - self._reward_buffer[battle]\n",
    "        self._reward_buffer[battle] = current_value\n",
    "        if use_neptune: run[f'N0: {self.n0} gamma: {self.gamma} reward_buffer'].log(current_value)\n",
    "        return to_return\n",
    "\n",
    "    # Calling reward_computing_helper\n",
    "    def compute_reward(self, battle) -> float:\n",
    "        return self.reward_computing_helper(battle, fainted_value=2, hp_value=1, victory_value=15)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "92c94ce3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# validation player\n",
    "\n",
    "class ValidationPlayer(PlayerMC_FA):\n",
    "    def __init__(self, battle_format, team, w, N, n0):\n",
    "        super().__init__(battle_format=battle_format, team=team)\n",
    "        self.w = w\n",
    "        self.N = N\n",
    "        self.n0 = n0\n",
    "        self.policy = self.update_epsilon_greedy_policy(self.w, self.n0, self.N)\n",
    "\n",
    "    def x(self, state, action):\n",
    "        state = np.array(state).astype(float)\n",
    "        return np.append(state, action)\n",
    "\n",
    "    # q^(S, A, W)\n",
    "    def q_approx(self, state, action, w):\n",
    "        state = np.array(state).astype(float)\n",
    "        \n",
    "        return np.dot(self.x(state, action), w)\n",
    "\n",
    "    # max(a, q^(S, a', W))\n",
    "    def max_q_approx(self, state, w):\n",
    "        state = np.array(state).astype(float)\n",
    "        return max(np.array([self.q_approx(state, action, w) for action in range(N_OUR_ACTIONS)]))\n",
    "    \n",
    "    def choose_move(self, battle):\n",
    "        state = self.embed_battle(battle)\n",
    "        # let's get the greedy action. Ties must be broken arbitrarily\n",
    "\n",
    "        \n",
    "        action = np.random.choice(self.action_space, p=self.policy(state))\n",
    "       \n",
    "\n",
    "        # if the selected action is not possible, perform a random move instead\n",
    "        if action == -1:\n",
    "            return ForfeitBattleOrder()\n",
    "        elif action < 4 and action < len(battle.available_moves) and not battle.force_switch:\n",
    "            return self.create_order(battle.available_moves[action])\n",
    "        elif 0 <= action - 4 < len(battle.available_switches):\n",
    "            return self.create_order(battle.available_switches[action - 4])\n",
    "        else:\n",
    "            return self.choose_random_move(battle)\n",
    "\n",
    "    def _battle_finished_callback(self, battle):\n",
    "        pass\n",
    "\n",
    "    # the embed battle is our state\n",
    "    # 11 factors: 4 moves base power, 4 moves multipliers, active pokemon and active opponent pokemon \n",
    "    def embed_battle(self, battle):\n",
    "        # -1 indicates that the move does not have a base power or not available\n",
    "\n",
    "        moves_base_power = -np.ones(4)\n",
    "        moves_dmg_multiplier = np.ones(4)\n",
    "        for i, move in enumerate(battle.available_moves):\n",
    "            moves_base_power[i] = (\n",
    "                move.base_power / 100\n",
    "            )  # Simple rescaling \n",
    "            if move.type:\n",
    "                moves_dmg_multiplier[i] = move.type.damage_multiplier(\n",
    "                    battle.opponent_active_pokemon.type_1,\n",
    "                    battle.opponent_active_pokemon.type_2,\n",
    "                )\n",
    "\n",
    "        # how many pokemons have fainted in each team\n",
    "        remaining_mon_team = (\n",
    "            len([mon for mon in battle.team.values() if mon.fainted]) / 6\n",
    "        )\n",
    "        \n",
    "        remaining_mon_opponent = (\n",
    "            len([mon for mon in battle.opponent_team.values() if mon.fainted]) / 6\n",
    "        )\n",
    "        \n",
    "        state = list()\n",
    "        #active_pokemon = [mon for mon in battle.team.values() if mon._active]   \n",
    "        state.append('{0:.2f}'.format(name_to_id(str(battle.active_pokemon).split(' ')[0])))\n",
    "        state.append('{0:.2f}'.format(name_to_id(str(battle.opponent_active_pokemon).split(' ')[0])))    \n",
    "        for move_base_power in moves_base_power:\n",
    "            state.append('{0:.2f}'.format(move_base_power))\n",
    "        for move_dmg_multiplier in moves_dmg_multiplier:\n",
    "            state.append('{0:.2f}'.format(move_dmg_multiplier))\n",
    "        state.append('{0:.2f}'.format(remaining_mon_team))\n",
    "        state.append('{0:.2f}'.format(remaining_mon_opponent))\n",
    "        # Convert to string so we can use as hash\n",
    "        return state\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "5e2940bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# global parameters\n",
    "\n",
    "# possible values for num_battles (number of episodes)\n",
    "n_battles_array = [10000]\n",
    "# exploration schedule from MC, i. e., epsilon(t) = N0 / (N0 + N(S(t)))\n",
    "n0_array = [0.0001, 0.001, 0.01]\n",
    "\n",
    "# possible values for gamma (discount factor)\n",
    "gamma_array = [0.75]\n",
    "\n",
    "\n",
    "list_of_params = [\n",
    "    {\n",
    "        'n_battles': n_battles,\n",
    "        'n0': n0,\n",
    "        'gamma': gamma\n",
    "    } for n_battles, n0, gamma in product(n_battles_array, n0_array, gamma_array)\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04fab331",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "fe309652",
   "metadata": {},
   "outputs": [],
   "source": [
    "# json helper functions\n",
    "\n",
    "def save_array_to_json(path_dir, filename, data):\n",
    "    if not os.path.exists(path_dir):\n",
    "        os.makedirs(path_dir)\n",
    "    full_filename = path_dir + \"/\" + filename\n",
    "    # write\n",
    "    with open(full_filename, \"w\") as file:\n",
    "        json.dump(data if isinstance(data, list) else data.tolist(), file)\n",
    "        file.close()\n",
    "\n",
    "\n",
    "def save_dict_to_json(path_dir, filename, value):\n",
    "    if not os.path.exists(path_dir):\n",
    "        os.makedirs(path_dir)\n",
    "    full_filename = path_dir + \"/\" + filename\n",
    "    file = open(full_filename, \"w\")\n",
    "    value_dict = dict()\n",
    "    for key in value:\n",
    "        value_dict[key] = value[key].tolist()\n",
    "    json.dump(value_dict, file)\n",
    "    file.close()\n",
    "\n",
    "\n",
    "def read_array_from_json(path_dir, filename):\n",
    "    full_filename = path_dir + \"/\" + filename\n",
    "    if not os.path.exists(full_filename):\n",
    "        return None\n",
    "    file = open(full_filename, \"r\")\n",
    "    data = json.load(file)\n",
    "    file.close()\n",
    "    return data\n",
    "\n",
    "\n",
    "def read_dict_from_json(path_dir, filename):\n",
    "    full_filename = path_dir + \"/\" + filename\n",
    "    if not os.path.exists(full_filename):\n",
    "        return None\n",
    "    file = open(full_filename, \"r\")\n",
    "    data = json.load(file)\n",
    "    for key in data:\n",
    "        data[key] = np.array(data[key])\n",
    "    file.close()\n",
    "    return data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "42a1336f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.0001, gamma=0.75, wins=8270, winning %=82.70, total time=1629.61 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\leolellisr\\anaconda3\\envs\\poke_env\\lib\\site-packages\\ipykernel_launcher.py:69: RuntimeWarning: overflow encountered in multiply\n",
      "C:\\Users\\leolellisr\\anaconda3\\envs\\poke_env\\lib\\site-packages\\ipykernel_launcher.py:69: RuntimeWarning: invalid value encountered in multiply\n",
      "C:\\Users\\leolellisr\\anaconda3\\envs\\poke_env\\lib\\site-packages\\ipykernel_launcher.py:69: RuntimeWarning: invalid value encountered in add\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training: num battles (episodes)=10000, N0=0.0010, gamma=0.75, wins=8302, winning %=83.02, total time=1726.06 sec\n",
      "training: num battles (episodes)=10000, N0=0.0100, gamma=0.75, wins=8308, winning %=83.08, total time=1724.92 sec\n"
     ]
    }
   ],
   "source": [
    "# main (let's battle!)\n",
    "\n",
    "# training\n",
    "async def do_battle_training():\n",
    "    for params in list_of_params:\n",
    "        start = time.time()\n",
    "        params['player'] = MCPlayer(battle_format=\"gen8ou\", team=OUR_TEAM, n0=params['n0'], gamma=params['gamma'])\n",
    "        params['opponent'] = MaxDamagePlayer(battle_format=\"gen8ou\", team=OP_TEAM)\n",
    "        await params['player'].battle_against(opponent=params['opponent'], n_battles=params['n_battles'])\n",
    "        if debug:\n",
    "            print(\"training: num battles (episodes)=%d, N0=%.4f, gamma=%.2f, wins=%d, winning %%=%.2f, total time=%s sec\" %\n",
    "                  (\n",
    "                      params['n_battles'],\n",
    "                      round(params['n0'], 4),\n",
    "                      round(params['gamma'], 2),\n",
    "                      params['player'].n_won_battles,\n",
    "                      round((params['player'].n_won_battles / params['n_battles']) * 100, 2),\n",
    "                      round(time.time() - start, 2)\n",
    "                  ))\n",
    "\n",
    "        # save w to json file\n",
    "        if save_to_json_file:\n",
    "            today_s = str(date.today())\n",
    "            n_battle_s = str(params['n_battles'])\n",
    "            n0_s = str(round(params['n0'], 4))\n",
    "            gamma_s = str(round(params['gamma'], 4))\n",
    "            winning_percentage_s = str(round((params['player'].n_won_battles / params['n_battles']) * 100, 2))\n",
    "\n",
    "            filename = \"W_\" + today_s + \"_\" + n_battle_s + \"_\" + n0_s + \"_\" + gamma_s + \"_\" + winning_percentage_s+\".json\"  \n",
    "            save_array_to_json(\"./MC_Control_FA_w\", filename, params['player'].w)\n",
    "            \n",
    "            filename = \"N_\" + today_s + \"_\" + n_battle_s + \"_\" + n0_s + \"_\" + gamma_s + \"_\" + winning_percentage_s+\".json\"           \n",
    "            save_dict_to_json(\"./MC_Control_FA_N\", filename, params['player'].N)\n",
    "        # statistics: key: \"n_battles, n0, alpha, gamma\", values: list of win or lose\n",
    "        key = str(params['n_battles']) + \"_\" + str(round(params['n0'], 4)) + \"_\" + str(round(params['gamma'], 2))\n",
    "        winning_status = list()\n",
    "        for battle in params['player']._battles.values():\n",
    "            if battle.won:\n",
    "                winning_status.append(True)\n",
    "            else:\n",
    "                winning_status.append(False)\n",
    "        # save statistics json file (append)\n",
    "        data = dict()\n",
    "        data[key] = winning_status\n",
    "        save_dict_to_json(\"./MC_Control_FA_statistics\", \"statistics.json\", data)\n",
    "\n",
    "\n",
    "loop = asyncio.get_event_loop()\n",
    "loop.run_until_complete(loop.create_task(do_battle_training()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "154e209f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44862642",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0e7fcb58",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plotting helper functions\n",
    "\n",
    "def plot_2d(path, title, x_label, x_array, y_label, y_array):\n",
    "    # print(\"plotting %s\" % title)\n",
    "    # set labels and plot surface\n",
    "    figure = matplotlib.pyplot.figure(figsize=(20, 10))\n",
    "    ax = figure.gca()\n",
    "    ax.set_title(title)\n",
    "    ax.set_xlabel(x_label)\n",
    "    ax.set_ylabel(y_label)\n",
    "    ax.plot(x_array, y_array)\n",
    "    # pyplot.show()\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "    filename = path + \"/\" + title + \"_\" + x_label + \"_\" + y_label + \"_\" + \".pdf\"\n",
    "    figure.savefig(filename, dpi=figure.dpi)\n",
    "    pyplot.close(figure)\n",
    "    # print(\"done!\")\n",
    "\n",
    "\n",
    "def plot_3d(path, title, x_label, x_array, y_label, y_array, z_label, z_array):\n",
    "    # print(\"plotting %s\" % title)\n",
    "    xyz = {'x': x_array, 'y': y_array, 'z': z_array}\n",
    "    df = pd.DataFrame(xyz, index=range(len(xyz['x'])))\n",
    "    xv, yv = np.meshgrid(x_array, y_array)\n",
    "    zv = griddata((df['x'], df['y']), df['z'], (xv, yv), method='nearest')\n",
    "    # set labels and plot surface\n",
    "    figure = matplotlib.pyplot.figure(figsize=(20, 10))\n",
    "    ax = figure.gca(projection='3d')\n",
    "    ax.set_title(title)\n",
    "    ax.set_xlabel(x_label)\n",
    "    ax.set_ylabel(y_label)\n",
    "    ax.set_zlabel(z_label)\n",
    "    surface = ax.plot_surface(xv, yv, zv, rstride=1, cstride=1, cmap=matplotlib.cm.coolwarm, linewidth=0,\n",
    "                              antialiased=False)\n",
    "    figure.colorbar(surface)\n",
    "    # pyplot.show()\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "    filename = path + \"/\" + title + \".pdf\"\n",
    "    figure.savefig(filename, dpi=figure.dpi)\n",
    "    pyplot.close(figure)\n",
    "    # print(\"done!\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6c97a03b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot additional statistics\n",
    "\n",
    "def plot_statistics_json(path_dir, filename=\"statistics.json\"):\n",
    "    # plots from statistics.json\n",
    "    statistics = read_dict_from_json(path_dir, filename)\n",
    "    # win/lost vs. episode number\n",
    "    for key in statistics.keys():\n",
    "        key_elements = key.split(\"_\")\n",
    "        n_battles = key_elements[0]\n",
    "        n0 = key_elements[1]\n",
    "        gamma = key_elements[2]\n",
    "        value = statistics[key]\n",
    "        plot_2d(path=\"./MC_Control_FA_plot\",\n",
    "                title=\"acc_victories_n_battles_\" + n_battles + \"_N0_\" + n0 + \"_gamma_\" + gamma,\n",
    "                x_label=\"episodes\",\n",
    "                x_array=np.array(range(0, len(value))),\n",
    "                y_label=\"acc victory\",\n",
    "                y_array=np.cumsum(value).astype(np.int))\n",
    "\n",
    "    # winning % by set of parameters\n",
    "    n_battles = \"\"\n",
    "    alpha = \"\"\n",
    "    x_values = []\n",
    "    y_values = []\n",
    "    z_values = []\n",
    "    for key in statistics.keys():\n",
    "        key_elements = key.split(\"_\")\n",
    "        n_battles = key_elements[0]\n",
    "        n0 = key_elements[1]\n",
    "        gamma = key_elements[2]\n",
    "        value = statistics[key]\n",
    "        x_values.append(n0)\n",
    "        y_values.append(gamma)\n",
    "        z_values.append(value.count(True) / len(value))\n",
    "    plot_3d(path=\"./MC_Control_FA_plot\",\n",
    "            title=\"winning_percentage_n_battles_\" + n_battles,\n",
    "            x_label=\"N0\",\n",
    "            x_array=np.array(x_values).astype(np.float),\n",
    "            y_label=\"gamma\",\n",
    "            y_array=np.array(y_values).astype(np.float),\n",
    "            z_label=\"winning %\",\n",
    "            z_array=np.array(z_values))\n",
    "\n",
    "\n",
    "# plots from statistics\n",
    "plot_statistics_json(\"./MC_Control_FA_statistics\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "335a07d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-03 15:09:58,878 - ValidationPlaye 12 - ERROR - Unhandled exception raised while handling message:\n",
      ">battle-gen8ou-2598291\n",
      "|\n",
      "|t:|1635962998\n",
      "|start\n",
      "|switch|p1a: Venusaur|Venusaur, M|301/301\n",
      "|switch|p2a: Umbreon|Umbreon, F|100/100\n",
      "|turn|1\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\leolellisr\\anaconda3\\envs\\poke_env\\lib\\site-packages\\poke_env\\player\\player_network_interface.py\", line 131, in _handle_message\n",
      "    await self._handle_battle_message(split_messages)\n",
      "  File \"S:\\poke_env\\src\\playerMC_FA.py\", line 358, in _handle_battle_message\n",
      "    self.logger.critical(\"Unexpected error message: %s\", split_message)\n",
      "  File \"S:\\poke_env\\src\\playerMC_FA.py\", line 379, in _handle_battle_request\n",
      "    return\n",
      "  File \"<ipython-input-57-58d6eb9f1421>\", line 35, in choose_move\n",
      "    action = np.random.choice(self.action_space, p=self.policy(state))\n",
      "  File \"S:\\poke_env\\src\\playerMC_FA.py\", line 185, in policy_fn\n",
      "    def policy_fn(state):\n",
      "  File \"S:\\poke_env\\src\\playerMC_FA.py\", line 185, in <listcomp>\n",
      "    def policy_fn(state):\n",
      "  File \"S:\\poke_env\\src\\playerMC_FA.py\", line 179, in q_approx_fn\n",
      "    print(w)\n",
      "  File \"<__array_function__ internals>\", line 6, in dot\n",
      "TypeError: unsupported operand type(s) for *: 'float' and 'NoneType'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n",
      "<class 'numpy.ndarray'>\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Task exception was never retrieved\n",
      "future: <Task finished coro=<PlayerNetwork._handle_message() done, defined at C:\\Users\\leolellisr\\anaconda3\\envs\\poke_env\\lib\\site-packages\\poke_env\\player\\player_network_interface.py:117> exception=TypeError(\"unsupported operand type(s) for *: 'float' and 'NoneType'\",)>\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\leolellisr\\anaconda3\\envs\\poke_env\\lib\\asyncio\\tasks.py\", line 180, in _step\n",
      "    result = coro.send(None)\n",
      "  File \"C:\\Users\\leolellisr\\anaconda3\\envs\\poke_env\\lib\\site-packages\\poke_env\\player\\player_network_interface.py\", line 177, in _handle_message\n",
      "    raise exception\n",
      "  File \"C:\\Users\\leolellisr\\anaconda3\\envs\\poke_env\\lib\\site-packages\\poke_env\\player\\player_network_interface.py\", line 131, in _handle_message\n",
      "    await self._handle_battle_message(split_messages)\n",
      "  File \"S:\\poke_env\\src\\playerMC_FA.py\", line 358, in _handle_battle_message\n",
      "    self.logger.critical(\"Unexpected error message: %s\", split_message)\n",
      "  File \"S:\\poke_env\\src\\playerMC_FA.py\", line 379, in _handle_battle_request\n",
      "    return\n",
      "  File \"<ipython-input-57-58d6eb9f1421>\", line 35, in choose_move\n",
      "    action = np.random.choice(self.action_space, p=self.policy(state))\n",
      "  File \"S:\\poke_env\\src\\playerMC_FA.py\", line 185, in policy_fn\n",
      "    def policy_fn(state):\n",
      "  File \"S:\\poke_env\\src\\playerMC_FA.py\", line 185, in <listcomp>\n",
      "    def policy_fn(state):\n",
      "  File \"S:\\poke_env\\src\\playerMC_FA.py\", line 179, in q_approx_fn\n",
      "    print(w)\n",
      "  File \"<__array_function__ internals>\", line 6, in dot\n",
      "TypeError: unsupported operand type(s) for *: 'float' and 'NoneType'\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-59-c399c8f303b4>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     32\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0muse_validation\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     33\u001b[0m     \u001b[0mloop\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0masyncio\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_event_loop\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 34\u001b[1;33m     \u001b[0mloop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun_until_complete\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mloop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate_task\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdo_battle_validation\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"./MC_Control_FA_w\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"./MC_Control_FA_N\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\anaconda3\\envs\\poke_env\\lib\\site-packages\\nest_asyncio.py\u001b[0m in \u001b[0;36mrun_until_complete\u001b[1;34m(self, future)\u001b[0m\n\u001b[0;32m     62\u001b[0m                 \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_log_destroy_pending\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     63\u001b[0m             \u001b[1;32mwhile\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdone\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 64\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_run_once\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     65\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_stopping\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     66\u001b[0m                     \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\poke_env\\lib\\site-packages\\nest_asyncio.py\u001b[0m in \u001b[0;36m_run_once\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     85\u001b[0m             \u001b[1;32melse\u001b[0m \u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscheduled\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_when\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mnow\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m86400\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mscheduled\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     86\u001b[0m             else None)\n\u001b[1;32m---> 87\u001b[1;33m         \u001b[0mevent_list\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_selector\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mselect\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     88\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_process_events\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mevent_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     89\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\poke_env\\lib\\selectors.py\u001b[0m in \u001b[0;36mselect\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    321\u001b[0m         \u001b[0mready\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    322\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 323\u001b[1;33m             \u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_select\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_readers\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_writers\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    324\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mInterruptedError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    325\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mready\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\envs\\poke_env\\lib\\selectors.py\u001b[0m in \u001b[0;36m_select\u001b[1;34m(self, r, w, _, timeout)\u001b[0m\n\u001b[0;32m    312\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplatform\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'win32'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    313\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0m_select\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 314\u001b[1;33m             \u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mselect\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mselect\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    315\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    316\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# validation\n",
    "\n",
    "async def do_battle_validation(path_dir,path_dirN):\n",
    "    # read from json\n",
    "    for filenamew, filenameN in zip(os.listdir(path_dir), os.listdir(path_dirN)):\n",
    "        # learned feature vector\n",
    "        w = np.array(read_array_from_json(path_dir, filename))\n",
    "        N = read_dict_from_json(path_dirN, filenameN)\n",
    "        # params: n_battles, n0, gamma\n",
    "        params = filename.split(\"_\")\n",
    "        n_battles = int(params[2])\n",
    "        n0 = float(params[3])\n",
    "        gamma = float(params[4])\n",
    "\n",
    "        # validation (play 1/3 of the battles using Q-learned table)\n",
    "        start = time.time()\n",
    "        validation_player = ValidationPlayer(battle_format=\"gen8ou\", team=OUR_TEAM, w=w, N=N, n0=n0)\n",
    "        opponent = MaxDamagePlayer(battle_format=\"gen8ou\", team=OP_TEAM)\n",
    "        n_battles_validation = int(n_battles / 3)\n",
    "        await validation_player.battle_against(opponent=opponent, n_battles=n_battles_validation)\n",
    "        print(\"validation: num battles (episodes)=%d, N0=%.4f, gamma=%.2f, wins=%d, winning %%=%.2f, total time=%s sec\" %\n",
    "              (\n",
    "                  n_battles_validation,\n",
    "                  n0,\n",
    "                  gamma,\n",
    "                  validation_player.n_won_battles,\n",
    "                  round((validation_player.n_won_battles / n_battles_validation) * 100, 2),\n",
    "                  round(time.time() - start, 2)\n",
    "              ))\n",
    "\n",
    "\n",
    "if use_validation:\n",
    "    loop = asyncio.get_event_loop()\n",
    "    loop.run_until_complete(loop.create_task(do_battle_validation(\"./MC_Control_FA_w\", \"./MC_Control_FA_N\")))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "5aa3e327",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation: num battles (episodes)=3333, N0=0.0001, gamma=0.75, wins=2771, winning %=83.14, total time=521.1 sec\n",
      "validation: num battles (episodes)=3333, N0=0.0010, gamma=0.75, wins=2796, winning %=83.89, total time=579.7 sec\n",
      "validation: num battles (episodes)=3333, N0=0.0100, gamma=0.75, wins=2778, winning %=83.35, total time=807.72 sec\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-11-03 15:55:12,505 - ValidationPlayer 1 - WARNING - Trying to login as ValidationPlayer 1, showdown returned  ValidationPlayer 1@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 15:55:12,537 - ValidationPlayer 2 - WARNING - Trying to login as ValidationPlayer 2, showdown returned  ValidationPlayer 2@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 15:55:12,540 - ValidationPlayer 4 - WARNING - Trying to login as ValidationPlayer 4, showdown returned  ValidationPlayer 4@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 15:55:12,542 - MaxDamagePlayer 5 - WARNING - Trying to login as MaxDamagePlayer 5, showdown returned  MaxDamagePlayer 5@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 15:55:12,545 - ValidationPlayer 3 - WARNING - Trying to login as ValidationPlayer 3, showdown returned  ValidationPlayer 3@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 15:55:12,549 - MaxDamagePlayer 4 - WARNING - Trying to login as MaxDamagePlayer 4, showdown returned  MaxDamagePlayer 4@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 15:55:12,552 - MaxDamagePlayer 6 - WARNING - Trying to login as MaxDamagePlayer 6, showdown returned  MaxDamagePlayer 6@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 15:55:12,563 - ValidationPlayer 5 - WARNING - Trying to login as ValidationPlayer 5, showdown returned  ValidationPlayer 5@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 15:55:12,565 - MaxDamagePlayer 7 - WARNING - Trying to login as MaxDamagePlayer 7, showdown returned  MaxDamagePlayer 7@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 15:55:12,568 - MaxDamagePlayer 8 - WARNING - Trying to login as MaxDamagePlayer 8, showdown returned  MaxDamagePlayer 8@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,525 - ValidationPlayer 6 - WARNING - Trying to login as ValidationPlayer 6, showdown returned  ValidationPlayer 6@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,670 - ValidationPlayer 8 - WARNING - Trying to login as ValidationPlayer 8, showdown returned  ValidationPlayer 8@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,677 - ValidationPlayer 7 - WARNING - Trying to login as ValidationPlayer 7, showdown returned  ValidationPlayer 7@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,684 - MaxDamagePlayer 13 - WARNING - Trying to login as MaxDamagePlayer 13, showdown returned  MaxDamagePlayer 13@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,692 - MaxDamagePlayer 9 - WARNING - Trying to login as MaxDamagePlayer 9, showdown returned  MaxDamagePlayer 9@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,697 - ValidationPlaye 10 - WARNING - Trying to login as ValidationPlaye 10, showdown returned  ValidationPlaye 10@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,704 - MaxDamagePlayer 11 - WARNING - Trying to login as MaxDamagePlayer 11, showdown returned  MaxDamagePlayer 11@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,710 - MaxDamagePlayer 14 - WARNING - Trying to login as MaxDamagePlayer 14, showdown returned  MaxDamagePlayer 14@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,715 - MaxDamagePlayer 12 - WARNING - Trying to login as MaxDamagePlayer 12, showdown returned  MaxDamagePlayer 12@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,724 - ValidationPlaye 11 - WARNING - Trying to login as ValidationPlaye 11, showdown returned  ValidationPlaye 11@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,732 - ValidationPlayer 9 - WARNING - Trying to login as ValidationPlayer 9, showdown returned  ValidationPlayer 9@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,741 - MaxDamagePlayer 15 - WARNING - Trying to login as MaxDamagePlayer 15, showdown returned  MaxDamagePlayer 15@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,744 - MaxDamagePlayer 10 - WARNING - Trying to login as MaxDamagePlayer 10, showdown returned  MaxDamagePlayer 10@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,748 - MaxDamagePlayer 17 - WARNING - Trying to login as MaxDamagePlayer 17, showdown returned  MaxDamagePlayer 17@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,751 - MaxDamagePlayer 16 - WARNING - Trying to login as MaxDamagePlayer 16, showdown returned  MaxDamagePlayer 16@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,758 - ValidationPlaye 12 - WARNING - Trying to login as ValidationPlaye 12, showdown returned  ValidationPlaye 12@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,762 - ValidationPlaye 14 - WARNING - Trying to login as ValidationPlaye 14, showdown returned  ValidationPlaye 14@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n",
      "2021-11-03 16:25:12,765 - ValidationPlaye 13 - WARNING - Trying to login as ValidationPlaye 13, showdown returned  ValidationPlaye 13@! - this might prevent future actions from this agent. Changing the agent's username might solve this problem.\n"
     ]
    }
   ],
   "source": [
    "# validation\n",
    "\n",
    "async def do_battle_validation_params(params):\n",
    "    # read from json\n",
    "    for parm in params:\n",
    "        # learned feature vector\n",
    "        w = parm['player'].w\n",
    "        N = parm['player'].N\n",
    "        # params: n_battles, n0, gamma\n",
    "        n_battles = parm['n_battles']\n",
    "        n0 = parm['n0']\n",
    "        gamma = parm['gamma']\n",
    "\n",
    "        # validation (play 1/3 of the battles using Q-learned table)\n",
    "        start = time.time()\n",
    "        validation_player = ValidationPlayer(battle_format=\"gen8ou\", team=OUR_TEAM, w=w, N=N, n0=n0)\n",
    "        opponent = MaxDamagePlayer(battle_format=\"gen8ou\", team=OP_TEAM)\n",
    "        n_battles_validation = int(n_battles / 3)\n",
    "        await validation_player.battle_against(opponent=opponent, n_battles=n_battles_validation)\n",
    "        print(\"validation: num battles (episodes)=%d, N0=%.4f, gamma=%.2f, wins=%d, winning %%=%.2f, total time=%s sec\" %\n",
    "              (\n",
    "                  n_battles_validation,\n",
    "                  n0,\n",
    "                  gamma,\n",
    "                  validation_player.n_won_battles,\n",
    "                  round((validation_player.n_won_battles / n_battles_validation) * 100, 2),\n",
    "                  round(time.time() - start, 2)\n",
    "              ))\n",
    "\n",
    "\n",
    "if use_validation:\n",
    "    loop = asyncio.get_event_loop()\n",
    "    loop.run_until_complete(loop.create_task(do_battle_validation_params(list_of_params)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7d0d385",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60409c93",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}