MC_Control_deterministic.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd46ab91",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Notebook for training and testing a Player in Pokemon Showdown using Monte Carlo "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76ea6ed2",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Max Player Performance: 57,8 % \n",
    "------\n",
    "neptune\n",
    "Action space: 4 moves + 5 switches\n",
    "poke-env installed in C:\\\\Users\\\\-\\\\anaconda3\\\\envs\\\\poke_env\\\\lib\\\\site-packages\n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5ab8d4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7340ee13",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://app.neptune.ai/leolellisr/rl-pokeenv/e/RLPOK-102\n",
      "Remember to stop your run once you’ve finished logging your metadata (https://docs.neptune.ai/api-reference/run#stop). It will be stopped automatically only when the notebook kernel/interactive console is terminated.\n"
     ]
    }
   ],
   "source": [
    "# Imports\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import sys\n",
    "from gym import spaces\n",
    "import asyncio\n",
    "import time\n",
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "from src.playerMC import Player as PlayerMC\n",
    "\n",
    "from poke_env.player.player import Player \n",
    "from poke_env.player.random_player import RandomPlayer\n",
    "\n",
    "from poke_env.environment.abstract_battle import AbstractBattle\n",
    "\n",
    "import matplotlib\n",
    "import pandas as pd\n",
    "from collections import namedtuple\n",
    "from matplotlib import pyplot as plt\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "\n",
    "np.random.seed(0)\n",
    "#loop = asyncio.get_event_loop()\n",
    "use_neptune = True\n",
    "if use_neptune:\n",
    "    import neptune.new as neptune\n",
    "    run = neptune.init(project='leolellisr/rl-pokeenv',\n",
    "                       api_token='eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLm5lcHR1bmUuYWkiLCJhcGlfdXJsIjoiaHR0cHM6Ly9hcHAubmVwdHVuZS5haSIsImFwaV9rZXkiOiI1NjY1YmJkZi1hYmM5LTQ3M2QtOGU1ZC1iZTFlNWY4NjE1NDQifQ==')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "903ea52b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def name_to_id(name):\n",
    "    if(name == 'turtonator'): return 0\n",
    "    if(name == 'lapras'): return 1 \n",
    "    if(name == 'armaldo'): return 2 \n",
    "    if(name == 'drapion'): return 3 \n",
    "    if(name == 'kabutops'): return 4 \n",
    "    if(name == 'falinks'): return 5 \n",
    "    if(name == 'cloyster'): return 0\n",
    "    if(name == 'omastar'): return 1 \n",
    "    if(name == 'crustle'): return 2 \n",
    "    if(name == 'escavalier'): return 3 \n",
    "    if(name == 'drednaw'): return 4 \n",
    "    if(name == 'typenull'): return 5     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "139879a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MaxDamagePlayer(Player):\n",
    "    def choose_move(self, battle):\n",
    "        # If the player can attack, it will\n",
    "        if battle.available_moves:\n",
    "            # Finds the best move among available ones\n",
    "            best_move = max(battle.available_moves, key=lambda move: move.base_power)\n",
    "            return self.create_order(best_move)\n",
    "\n",
    "        # If no attack is available, a random switch will be made\n",
    "        else:\n",
    "            return self.choose_random_move(battle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f0508551",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MCPlayer(PlayerMC):\n",
    "    def choose_move(self, battle):\n",
    "        # In the 1st state of all we don't append yet;\n",
    "        # Other states: using previous state and action with actual reward (actual battle) \n",
    "        if self.previous_action != -10: self.episode.append((self.previous_state, self.previous_action, self.compute_reward(battle)))\n",
    "            \n",
    "        # Getting state s (-> embed battle)\n",
    "        s = self.embed_battle(battle)\n",
    "        # 1st move will be random wout policy    \n",
    "        if(self.aux == 0):\n",
    "            self.aux = 1\n",
    "            action = np.random.choice(self.action_space)\n",
    "\n",
    "        # Other moves have policy    \n",
    "        else: action = np.random.choice(self.action_space, p=self.policy(s))\n",
    "\n",
    "        # Saving action and state to append later. We can compute the reward only after the move    \n",
    "        self.previous_action = action\n",
    "        self.previous_state = s\n",
    "\n",
    "        # Choose move according to action index\n",
    "        if action == -1:\n",
    "            return ForfeitBattleOrder()\n",
    "        elif (\n",
    "            action < 4\n",
    "            and action < len(battle.available_moves)\n",
    "            and not battle.force_switch\n",
    "        ):\n",
    "            return self.create_order(battle.available_moves[action])\n",
    "        elif 0 <= action - 4 < len(battle.available_switches):\n",
    "            return self.create_order(battle.available_switches[action - 4])\n",
    "        else:\n",
    "            return self.choose_random_move(battle)\n",
    "        \n",
    "\n",
    "    # the embed battle is our state\n",
    "    # 10 factors: 4 moves base power, 4 moves multipliers, active pokemon and active opponent pokemon \n",
    "    def embed_battle(self, battle):\n",
    "        # -1 indicates that the move does not have a base power or not available\n",
    "        moves_base_power = -np.ones(4)\n",
    "        moves_dmg_multiplier = np.ones(4)\n",
    "        for i, move in enumerate(battle.available_moves):\n",
    "            moves_base_power[i] = (\n",
    "                move.base_power / 100\n",
    "            )  # Simple rescaling \n",
    "            if move.type:\n",
    "                moves_dmg_multiplier[i] = move.type.damage_multiplier(\n",
    "                    battle.opponent_active_pokemon.type_1,\n",
    "                    battle.opponent_active_pokemon.type_2,\n",
    "                )\n",
    "\n",
    "        # how many pokemons have fainted in each team\n",
    "        remaining_mon_team = (\n",
    "            len([mon for mon in battle.team.values() if mon.fainted]) / 6\n",
    "        )\n",
    "        remaining_mon_opponent = (\n",
    "            len([mon for mon in battle.opponent_team.values() if mon.fainted]) / 6\n",
    "        )\n",
    "\n",
    "        state = list()       \n",
    "        state.append('{0:.2f}'.format(name_to_id(str(battle.active_pokemon).split(' ')[0])))\n",
    "        state.append('{0:.2f}'.format(name_to_id(str(battle.opponent_active_pokemon).split(' ')[0])))    \n",
    "        for move_base_power in moves_base_power:\n",
    "            state.append('{0:.2f}'.format(move_base_power))\n",
    "        for move_dmg_multiplier in moves_dmg_multiplier:\n",
    "            state.append('{0:.2f}'.format(move_dmg_multiplier))\n",
    "        state.append('{0:.2f}'.format(remaining_mon_team))\n",
    "        state.append('{0:.2f}'.format(remaining_mon_opponent))\n",
    "        # Convert to string so we can use as hash\n",
    "        return str(state)\n",
    "\n",
    "        \n",
    "\n",
    "    # Computing rewards\n",
    "    def reward_computing_helper(\n",
    "        self,\n",
    "        battle: AbstractBattle,\n",
    "        *,\n",
    "        fainted_value: float = 0.15, \n",
    "        hp_value: float = 0.15,      \n",
    "        number_of_pokemons: int = 6,\n",
    "        starting_value: float = 0.0,\n",
    "        status_value: float = 0.15, \n",
    "        victory_value: float = 1.0\n",
    "    ) -> float:\n",
    "        \n",
    "        # 1st compute\n",
    "        if battle not in self._reward_buffer:\n",
    "            self._reward_buffer[battle] = starting_value\n",
    "        current_value = 0\n",
    "        \n",
    "        # Verify if pokemon have fainted or have status\n",
    "        for mon in battle.team.values():\n",
    "            current_value += mon.current_hp_fraction * hp_value\n",
    "            if mon.fainted:\n",
    "                current_value -= fainted_value\n",
    "            elif mon.status is not None:\n",
    "                current_value -= status_value\n",
    "\n",
    "        current_value += (number_of_pokemons - len(battle.team)) * hp_value\n",
    "\n",
    "        # Verify if opponent pokemon have fainted or have status\n",
    "        for mon in battle.opponent_team.values():\n",
    "            current_value -= mon.current_hp_fraction * hp_value\n",
    "            if mon.fainted:\n",
    "                current_value += fainted_value\n",
    "            elif mon.status is not None:\n",
    "                current_value += status_value\n",
    "\n",
    "        current_value -= (number_of_pokemons - len(battle.opponent_team)) * hp_value\n",
    "\n",
    "        # Verify if we won or lost\n",
    "        if battle.won:\n",
    "            current_value += victory_value\n",
    "        elif battle.lost:\n",
    "            current_value -= victory_value\n",
    "\n",
    "        # Value to return\n",
    "        to_return = current_value - self._reward_buffer[battle]\n",
    "        self._reward_buffer[battle] = current_value\n",
    "        if use_neptune: run[f'N0: {self.n0} gamma: {self.gamma} reward_buffer'].log(current_value)\n",
    "        return to_return\n",
    "    \n",
    "    \n",
    "    # Calling reward_computing_helper\n",
    "    def compute_reward(self, battle) -> float:\n",
    "        return self.reward_computing_helper(            \n",
    "            battle, fainted_value=2, hp_value=1, status_value=1, victory_value=15\n",
    "        )\n",
    "\n",
    "    # Battle (episode) has finished\n",
    "    def _battle_finished_callback(self, battle):\n",
    "        rewards = [reward for state,action,reward in self.episode]\n",
    "        states = [state for state,action,reward in self.episode]\n",
    "        actions = [action for state,action,reward in self.episode]\n",
    "        t_array = range(len(self.episode)+1)\n",
    "        # Computing Q and N\n",
    "        for idx, state in enumerate(states):\n",
    "            action = actions[idx]\n",
    "            if state not in self.visited_states:\n",
    "                self.N[state][action] += 1\n",
    "                returnGt = sum([reward*pow(self.gamma, t) for reward, t in zip(rewards[idx:], t_array[:(-idx-1)])]) \n",
    "\n",
    "                # incremental update of Q value is more efficient than keeping a record of all rewards\n",
    "                # and averaging after every new reward\n",
    "                # step-size: 1./N[state][action]\n",
    "                \n",
    "                self.Q[state][action] += (1/self.N[state][action])*(returnGt-self.Q[state][action]) \n",
    "                #run[f'N0: {self.n0} gamma: {self.gamma} q_value'].log(self.Q[state][action])\n",
    "                self.visited_states.append(state)\n",
    "                \n",
    "        self.visited_states = []\n",
    "        self.episode = []\n",
    "        if(self.aux == 1):\n",
    "            self.aux == 0\n",
    "            \n",
    "        # Define new policy with updated Q and N\n",
    "        self.policy = self.update_epsilon_greedy_policy(self.Q, self.n0, self.N)\n",
    "        if use_neptune: run[f'N0: {self.n0} gamma: {self.gamma} win_acc'].log(self.n_won_battles/len(self._reward_buffer))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f8ffcad2",
   "metadata": {},
   "outputs": [],
   "source": [
    "our_team = \"\"\"\n",
    "Turtonator @ White Herb  \n",
    "Ability: Shell Armor  \n",
    "EVs: 4 Atk / 252 SpA / 252 Spe  \n",
    "Rash Nature  \n",
    "- Flamethrower  \n",
    "- Dragon Pulse  \n",
    "- Earthquake  \n",
    "- Shell Smash  \n",
    "\n",
    "Lapras @ Leftovers  \n",
    "Ability: Shell Armor  \n",
    "EVs: 252 HP / 252 SpA / 4 SpD  \n",
    "Modest Nature  \n",
    "IVs: 0 Atk  \n",
    "- Freeze-Dry  \n",
    "- Surf  \n",
    "- Thunderbolt  \n",
    "- Toxic  \n",
    "\n",
    "Armaldo @ Assault Vest  \n",
    "Ability: Battle Armor  \n",
    "EVs: 252 HP / 252 Atk / 4 SpD  \n",
    "Adamant Nature  \n",
    "- Earthquake  \n",
    "- Knock Off  \n",
    "- X-Scissor  \n",
    "- Aqua Jet  \n",
    "\n",
    "Drapion @ Life Orb  \n",
    "Ability: Battle Armor  \n",
    "EVs: 252 Atk / 4 SpD / 252 Spe  \n",
    "Jolly Nature  \n",
    "- Poison Jab  \n",
    "- Knock Off  \n",
    "- Earthquake  \n",
    "- X-Scissor  \n",
    "\n",
    "Kabutops @ Aguav Berry  \n",
    "Ability: Battle Armor  \n",
    "EVs: 252 Atk / 4 SpD / 252 Spe  \n",
    "Jolly Nature  \n",
    "- Liquidation  \n",
    "- Leech Life  \n",
    "- Knock Off  \n",
    "- Swords Dance  \n",
    "\n",
    "Falinks @ Iapapa Berry  \n",
    "Ability: Battle Armor  \n",
    "EVs: 252 HP / 252 Atk / 4 SpD  \n",
    "Adamant Nature  \n",
    "- Close Combat  \n",
    "- Poison Jab  \n",
    "- Iron Head  \n",
    "- No Retreat  \n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "26f8ff07",
   "metadata": {},
   "outputs": [],
   "source": [
    "op_team = \"\"\"\n",
    "Cloyster @ Assault Vest  \n",
    "Ability: Shell Armor  \n",
    "EVs: 248 HP / 252 Atk / 8 SpA  \n",
    "Naughty Nature  \n",
    "- Icicle Spear  \n",
    "- Surf  \n",
    "- Tri Attack  \n",
    "- Poison Jab  \n",
    "\n",
    "Omastar @ White Herb  \n",
    "Ability: Shell Armor  \n",
    "EVs: 252 SpA / 4 SpD / 252 Spe  \n",
    "Modest Nature  \n",
    "IVs: 0 Atk  \n",
    "- Surf  \n",
    "- Ancient Power  \n",
    "- Earth Power  \n",
    "- Shell Smash  \n",
    "\n",
    "Crustle @ Leftovers  \n",
    "Ability: Shell Armor  \n",
    "EVs: 252 HP / 252 Atk / 4 SpD  \n",
    "Adamant Nature  \n",
    "- Earthquake  \n",
    "- Knock Off  \n",
    "- X-Scissor  \n",
    "- Stealth Rock  \n",
    "\n",
    "Escavalier @ Life Orb  \n",
    "Ability: Shell Armor  \n",
    "EVs: 248 HP / 252 Atk / 8 SpD  \n",
    "Adamant Nature  \n",
    "- Knock Off  \n",
    "- Swords Dance  \n",
    "- Iron Head  \n",
    "- Poison Jab  \n",
    "\n",
    "Drednaw @ Aguav Berry  \n",
    "Ability: Shell Armor  \n",
    "EVs: 248 HP / 252 Atk / 8 SpD  \n",
    "Adamant Nature  \n",
    "- Liquidation  \n",
    "- Earthquake  \n",
    "- Poison Jab  \n",
    "- Swords Dance  \n",
    "\n",
    "Type: Null @ Eviolite  \n",
    "Ability: Battle Armor  \n",
    "EVs: 252 HP / 252 Atk / 4 SpD  \n",
    "Adamant Nature  \n",
    "- Facade  \n",
    "- Sleep Talk  \n",
    "- Shadow Claw  \n",
    "- Rest  \n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1a77ea9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "342b0116",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Global variables\n",
    "\n",
    "# 3 sets of tests; 10k battles each\n",
    "n0_array = [0.0001, 0.001, 0.01]\n",
    "gamma_array = [0.75, 0.75, 0.75] \n",
    "n_battles = [10000 for n0 in n0_array] #all tests have 10k battles; Can be modified to different values (must have same shape)\n",
    "our_team_array = [our_team for n0 in n0_array] #all teams are the same; Can be modified to different teams (must have same shape)\n",
    "op_team_array = [op_team for n0 in n0_array] #all opponents are the same; Can be modified to different teams (must have same shape)\n",
    "\n",
    "tests =[ {'n0': n0,\n",
    "          'gamma': gamma,\n",
    "          'n_battles':n_battle,\n",
    "          'team':our_team,\n",
    "          'against':op_team,\n",
    "          'battle_format':\"gen8ou\"}\n",
    "        for n0,gamma,n_battle,our_team,op_team in zip(n0_array,gamma_array,n_battles,our_team_array,op_team_array)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "819e61be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tests)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79a39deb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e3a3f5c",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for test in tests:\n",
    "    start = time.time()\n",
    "    if use_neptune: run['params'] = test\n",
    "    test['opponent'] = MaxDamagePlayer(battle_format=\"gen8ou\", team=test['against'])\n",
    "    test['player'] = MCPlayer(battle_format=\"gen8ou\", team=test['team'], n0=test['n0'], gamma=test['gamma'])\n",
    "    await test['player'].battle_against(test['opponent'], n_battles=test['n_battles'])\n",
    "    \n",
    "    print(\n",
    "        \"Player with N0=%f and gamma=%f won %d / %d battles [this is %f percent and took %f seconds]\"\n",
    "        % (\n",
    "            round(test['n0'], 8), \n",
    "            round(test['gamma'], 8),\n",
    "            test['player'].n_won_battles,\n",
    "            len(test['player']._reward_buffer),\n",
    "            round(test['player'].n_won_battles/len(test['player']._reward_buffer)*100, 2),\n",
    "            round(time.time() - start, 2)\n",
    "        )\n",
    "    )\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61972d4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "if use_neptune: run.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c03595b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f0f41cd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import re\n",
    "from datetime import date\n",
    "today = date.today()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "779811b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper function: save to json file\n",
    "def save_to_json(path, params, name, value):\n",
    "    today_s = str(date.today())\n",
    "    n_battle_s = str(params['n_battles'])\n",
    "    n0_s = str(round(params['n0'], 8))\n",
    "    gamma_s = str(round(params['gamma'], 8))\n",
    "    winning_percentage_s = str(round((params['player'].n_won_battles / params['n_battles']) * 100, 2))\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "    filename = path + \"/\" + name + \"_MCControlDet_\" + today_s + \"_n_battles_\" + n_battle_s + \"_N0_\" + n0_s + \"_gamma_\" + gamma_s + \"_wining_\" + winning_percentage_s\n",
    "    filename=re.sub(r'(?<=\\d)[,\\.-]','',filename)+\".json\"\n",
    "    file = open(filename, \"w\")\n",
    "    value_dict = dict()\n",
    "    for key in value:\n",
    "        value_dict[key] = value[key].tolist()\n",
    "    json.dump(value_dict, file)\n",
    "    file.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "528e5203",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving Q and N to json files\n",
    "for params in tests:\n",
    "    save_to_json(\"./dump\", params, \"Q\", params['player'].Q)\n",
    "    save_to_json(\"./dump\", params, \"N\", params['player'].N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b9b6f94",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1870d72a",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ValidationPlayer(PlayerMC):\n",
    "    def __init__(self, battle_format, team, Q, N, n0):\n",
    "        super().__init__(battle_format=battle_format, team=team)\n",
    "        self.Q = Q\n",
    "        self.N = N\n",
    "        self.n0 = n0\n",
    "        self.policy = self.update_epsilon_greedy_policy(self.Q, self.n0, self.N)\n",
    "        \n",
    "    def choose_move(self, battle):\n",
    "        state = self.embed_battle(battle)\n",
    "        # let's get the greedy action. Ties must be broken arbitrarily\n",
    "        if state in self.Q.keys():\n",
    "            action = np.random.choice(self.action_space, p=self.policy(state))\n",
    "        else:\n",
    "            return self.choose_random_move(battle)\n",
    "\n",
    "        # if the selected action is not possible, perform a random move instead\n",
    "        if action == -1:\n",
    "            return ForfeitBattleOrder()\n",
    "        elif action < 4 and action < len(battle.available_moves) and not battle.force_switch:\n",
    "            return self.create_order(battle.available_moves[action])\n",
    "        elif 0 <= action - 4 < len(battle.available_switches):\n",
    "            return self.create_order(battle.available_switches[action - 4])\n",
    "        else:\n",
    "            return self.choose_random_move(battle)\n",
    "\n",
    "    def _battle_finished_callback(self, battle):\n",
    "        pass\n",
    "\n",
    "    # the embed battle is our state\n",
    "    # 11 factors: 4 moves base power, 4 moves multipliers, active pokemon and active opponent pokemon \n",
    "    def embed_battle(self, battle):\n",
    "        # -1 indicates that the move does not have a base power or not available\n",
    "\n",
    "        moves_base_power = -np.ones(4)\n",
    "        moves_dmg_multiplier = np.ones(4)\n",
    "        for i, move in enumerate(battle.available_moves):\n",
    "            moves_base_power[i] = (\n",
    "                move.base_power / 100\n",
    "            )  # Simple rescaling \n",
    "            if move.type:\n",
    "                moves_dmg_multiplier[i] = move.type.damage_multiplier(\n",
    "                    battle.opponent_active_pokemon.type_1,\n",
    "                    battle.opponent_active_pokemon.type_2,\n",
    "                )\n",
    "\n",
    "        # how many pokemons have fainted in each team\n",
    "        remaining_mon_team = (\n",
    "            len([mon for mon in battle.team.values() if mon.fainted]) / 6\n",
    "        )\n",
    "        \n",
    "        remaining_mon_opponent = (\n",
    "            len([mon for mon in battle.opponent_team.values() if mon.fainted]) / 6\n",
    "        )\n",
    "        \n",
    "        state = list()\n",
    "        #active_pokemon = [mon for mon in battle.team.values() if mon._active]   \n",
    "        state.append('{0:.2f}'.format(name_to_id(str(battle.active_pokemon).split(' ')[0])))\n",
    "        state.append('{0:.2f}'.format(name_to_id(str(battle.opponent_active_pokemon).split(' ')[0])))    \n",
    "        for move_base_power in moves_base_power:\n",
    "            state.append('{0:.2f}'.format(move_base_power))\n",
    "        for move_dmg_multiplier in moves_dmg_multiplier:\n",
    "            state.append('{0:.2f}'.format(move_dmg_multiplier))\n",
    "        state.append('{0:.2f}'.format(remaining_mon_team))\n",
    "        state.append('{0:.2f}'.format(remaining_mon_opponent))\n",
    "        # Convert to string so we can use as hash\n",
    "        return str(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "70ce9613",
   "metadata": {},
   "outputs": [],
   "source": [
    "Qarray = []\n",
    "directoryQ = r'S:\\poke_env\\dump\\211029\\qvalues'\n",
    "Narray = []\n",
    "directoryN = r'S:\\poke_env\\dump\\211029\\nvalues'\n",
    "for filenameQ in os.listdir(directoryQ):\n",
    "    Qjson_file = open(directoryQ+'/'+filenameQ,)\n",
    "    Qjson = json.load(Qjson_file)\n",
    "    for key in Qjson:\n",
    "        Qjson[key] = np.array(Qjson[key])\n",
    "    Qarray.append(Qjson)\n",
    "\n",
    "for filenameN in os.listdir(directoryN):\n",
    "    Njson_file = open(directoryN+'/'+filenameN,)\n",
    "    Njson = json.load(Njson_file)\n",
    "    for key in Njson:\n",
    "        Njson[key] = np.array(Njson[key])\n",
    "    Narray.append(Njson)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "85239b89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Player with N0=0.000100 and gamma=0.750000 won 3026 / 3333 battles [this is 0.910000 percent and took 841.880000 seconds]\n",
      "Player with N0=0.001000 and gamma=0.750000 won 3113 / 3333 battles [this is 0.930000 percent and took 825.550000 seconds]\n",
      "Player with N0=0.010000 and gamma=0.750000 won 3130 / 3333 battles [this is 0.940000 percent and took 762.680000 seconds]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "for test, qvalue, nvalue in zip(tests, Qarray, Narray):\n",
    "    start = time.time()\n",
    "    test['opponent'] = RandomPlayer(battle_format=\"gen8ou\", team=test['against'])\n",
    "    test['player_val'] = ValidationPlayer(battle_format=\"gen8ou\", team=test['team'], Q=qvalue, N=nvalue, n0=test['n0'])\n",
    "    await test['player_val'].battle_against(test['opponent'], n_battles=int(test['n_battles']/3))\n",
    "    \n",
    "    print(\n",
    "        \"Player with N0=%f and gamma=%f won %d / %d battles [this is %f percent and took %f seconds]\"\n",
    "        % (\n",
    "            round(test['n0'], 8),\n",
    "            round(test['gamma'], 8),\n",
    "            test['player_val'].n_won_battles,\n",
    "            int(test['n_battles']/3),\n",
    "            round(test['player_val'].n_won_battles/int(test['n_battles']/3), 2),\n",
    "            round(time.time() - start, 2)\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ea2899a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c959c45e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# state is 4 moves_base_power, 4 moves_dmg_multiplier, [remaining_mon_team, remaining_mon_opponent]\n",
    "# 3D graph: X: sum(moves_base_power * moves_dmg_multiplier), Y: remaining_mon_team - remaining_mon_opponent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26298722",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Data for plotting: Create value function from action-value function\n",
    "# by picking the best action at each state\n",
    "\n",
    "# x: index_pokemon*20+sum(moves_base_power * moves_dmg_multiplier)\n",
    "# y: remaining_mon_team - remaining_mon_opponent\n",
    "# z: value function\n",
    "\n",
    "v_array = []\n",
    "directoryQ = r'S:\\poke_env\\dump\\211029\\qvalues'\n",
    "\n",
    "for filenameQ in os.listdir(directoryQ):\n",
    "    Qjson_file = open(directoryQ+'/'+filenameQ,)\n",
    "    Qjson = json.load(Qjson_file)\n",
    "\n",
    "    for key in Qjson:\n",
    "        Qjson[key] = np.array(Qjson[key])\n",
    "        \n",
    "    z_values = []\n",
    "    x_values = []\n",
    "    y_values = []\n",
    "    #for state, actions in test['player_val'].Q.items():\n",
    "    for state, actions in Qjson.items():    \n",
    "        action_value = np.max(actions)\n",
    "        z_values.append(action_value)\n",
    "        state = re.sub(r\"[,!?><:'\\[\\]()@*~#]\",\"\", state)\n",
    "        key_float = [float(k) for k in state.split()]\n",
    "        x_emb = key_float[0]*20+key_float[1]*key_float[5]+key_float[2]*key_float[6]+key_float[3]*key_float[7]+key_float[4]*key_float[8]\n",
    "        x_values.append(x_emb)\n",
    "        y_emb = key_float[8]-key_float[9]\n",
    "        y_values.append(y_emb)\n",
    "        #V[x_emb,y_emb] = action_value\n",
    "    v_array.append((x_values, y_values, z_values))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "11e31b27",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from scipy.interpolate import griddata\n",
    "# create 1D-arrays from the 2D-arrays\n",
    "for vvalue, filenameQ in zip(v_array, os.listdir(directoryQ)):\n",
    "    print(filenameQ)\n",
    "    x_values, y_values, z_values = vvalue \n",
    "    z_plot = np.array(z_values).reshape(len(z_values),1)\n",
    "    x_plot = np.array(x_values)\n",
    "    y_plot = np.array(y_values)\n",
    "    xyz = {'x': x_plot, 'y': y_plot, 'z': np.array(z_values)}\n",
    "    df = pd.DataFrame(xyz, index=range(len(xyz['x']))) \n",
    "    x1 = np.linspace(df['x'].min(), df['x'].max(), len(df['x'].unique()))\n",
    "    y1 = np.linspace(df['y'].min(), df['y'].max(), len(df['y'].unique()))\n",
    "    x2, y2 = np.meshgrid(x1, y1)\n",
    "    z2 = griddata((df['x'], df['y']), df['z'], (x2, y2), method='nearest')\n",
    "\n",
    "    fig = plt.figure(figsize=(20, 10))\n",
    "    ax = fig.gca(projection='3d')\n",
    "    ax.set_xlabel('index_pokemon*20+sum(moves_base_power * moves_dmg_multiplier)')\n",
    "    ax.set_ylabel('remaining_mon_team - remaining_mon_opponent')\n",
    "    ax.set_zlabel('Value')\n",
    "    ax.set_title('Value - Index for x axis: 0 venusaur;  1*20 pikachuoriginal; 2*20 tauros, 3*20 sirfetchd, 4*20 blastoise, 5*20 charizard')\n",
    "\n",
    "    surf = ax.plot_surface(x2, y2, z2, rstride=1, cstride=1, cmap=matplotlib.cm.coolwarm,\n",
    "        linewidth=0, antialiased=False)\n",
    "    fig.colorbar(surf)\n",
    "\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae787283",
   "metadata": {},
   "outputs": [],
   "source": [
    "# x: sum(moves_base_power * moves_dmg_multiplier)\n",
    "# y: remaining_mon_team - remaining_mon_opponent\n",
    "# z: value function\n",
    "\n",
    "v_array = []\n",
    "directoryQ = r'S:\\poke_env\\dump\\211029\\qvalues'\n",
    "\n",
    "for filenameQ in os.listdir(directoryQ):\n",
    "    Qjson_file = open(directoryQ+'/'+filenameQ,)\n",
    "    Qjson = json.load(Qjson_file)\n",
    "\n",
    "    for key in Qjson:\n",
    "        Qjson[key] = np.array(Qjson[key])\n",
    "        \n",
    "    z_values = []\n",
    "    x_values = []\n",
    "    y_values = []\n",
    "\n",
    "    for state, actions in Qjson.items():    \n",
    "        action_value = np.max(actions)\n",
    "        z_values.append(action_value)\n",
    "        state = re.sub(r\"[,!?><:'\\[\\]()@*~#]\",\"\", state)\n",
    "        key_float = [float(k) for k in state.split()]\n",
    "        x_emb = key_float[1]*key_float[5]+key_float[2]*key_float[6]+key_float[3]*key_float[7]+key_float[4]*key_float[8]\n",
    "        x_values.append(x_emb)\n",
    "        y_emb = key_float[8]-key_float[9]\n",
    "        y_values.append(y_emb)\n",
    "        #V[x_emb,y_emb] = action_value\n",
    "    v_array.append((x_values, y_values, z_values))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15ce723b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from scipy.interpolate import griddata\n",
    "# create 1D-arrays from the 2D-arrays\n",
    "for vvalue, filenameQ in zip(v_array, os.listdir(directoryQ)):\n",
    "    print(filenameQ)\n",
    "    x_values, y_values, z_values = vvalue \n",
    "    z_plot = np.array(z_values).reshape(len(z_values),1)\n",
    "    x_plot = np.array(x_values)\n",
    "    y_plot = np.array(y_values)\n",
    "    xyz = {'x': x_plot, 'y': y_plot, 'z': np.array(z_values)}\n",
    "    df = pd.DataFrame(xyz, index=range(len(xyz['x']))) \n",
    "    x1 = np.linspace(df['x'].min(), df['x'].max(), len(df['x'].unique()))\n",
    "    y1 = np.linspace(df['y'].min(), df['y'].max(), len(df['y'].unique()))\n",
    "    x2, y2 = np.meshgrid(x1, y1)\n",
    "    z2 = griddata((df['x'], df['y']), df['z'], (x2, y2), method='nearest')\n",
    "\n",
    "    fig = plt.figure(figsize=(20, 10))\n",
    "    ax = fig.gca(projection='3d')\n",
    "    ax.set_xlabel('sum(moves_base_power * moves_dmg_multiplier)')\n",
    "    ax.set_ylabel('remaining_mon_team - remaining_mon_opponent')\n",
    "    ax.set_zlabel('Value')\n",
    "    ax.set_title('Value - No Index')\n",
    "\n",
    "    surf = ax.plot_surface(x2, y2, z2, rstride=1, cstride=1, cmap=matplotlib.cm.coolwarm,\n",
    "        linewidth=0, antialiased=False)\n",
    "    fig.colorbar(surf)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0065d89e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# x: (remaining_mon_team - remaining_mon_opponent)*sum(moves_base_power * moves_dmg_multiplier)\n",
    "# y: action\n",
    "# z: value function\n",
    "\n",
    "v_array = []\n",
    "#for test in tests:\n",
    "directoryQ = r'S:\\poke_env\\dump\\211029\\qvalues'\n",
    "\n",
    "for filenameQ in os.listdir(directoryQ):\n",
    "    Qjson_file = open(directoryQ+'/'+filenameQ,)\n",
    "    Qjson = json.load(Qjson_file)\n",
    "\n",
    "    for key in Qjson:\n",
    "        Qjson[key] = np.array(Qjson[key])\n",
    "        \n",
    "    z_values = []\n",
    "    x_values = []\n",
    "    y_values = []\n",
    "    for state, actions in Qjson.items(): \n",
    "        \n",
    "        action_value = np.max(actions)\n",
    "        z_values.append(action_value)\n",
    "        state = re.sub(r\"[,!?><:'\\[\\]()@*~#]\",\"\", state)\n",
    "        key_float = [float(k) for k in state.split()]\n",
    "        x_emb = (key_float[8]-key_float[9])*(key_float[1]*key_float[5]+key_float[2]*key_float[6]+key_float[3]*key_float[7]+key_float[4]*key_float[8])\n",
    "        x_values.append(x_emb)\n",
    "        y_emb = np.argmax(actions)\n",
    "        y_values.append(y_emb)\n",
    "        V[x_emb,y_emb] = action_value\n",
    "    v_array.append((x_values, y_values, z_values))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f32b72c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from scipy.interpolate import griddata\n",
    "# create 1D-arrays from the 2D-arrays\n",
    "for vvalue, filenameQ in zip(v_array, os.listdir(directoryQ)):\n",
    "    print(filenameQ)\n",
    "    x_values, y_values, z_values = vvalue \n",
    "    z_plot = np.array(z_values).reshape(len(z_values),1)\n",
    "    x_plot = np.array(x_values)\n",
    "    y_plot = np.array(y_values)\n",
    "    xyz = {'x': x_plot, 'y': y_plot, 'z': np.array(z_values)}\n",
    "    df = pd.DataFrame(xyz, index=range(len(xyz['x']))) \n",
    "    x1 = np.linspace(df['x'].min(), df['x'].max(), len(df['x'].unique()))\n",
    "    y1 = np.linspace(df['y'].min(), df['y'].max(), len(df['y'].unique()))\n",
    "    x2, y2 = np.meshgrid(x1, y1)\n",
    "    z2 = griddata((df['x'], df['y']), df['z'], (x2, y2), method='nearest')\n",
    "\n",
    "    fig = plt.figure(figsize=(20, 10))\n",
    "    ax = fig.gca(projection='3d')\n",
    "    ax.set_xlabel('(remaining_mon_team - remaining_mon_opponent)*sum(moves_base_power * moves_dmg_multiplier)')\n",
    "    ax.set_ylabel('action')\n",
    "    ax.set_zlabel('Value')\n",
    "    ax.set_title('Value Function - No index')\n",
    "\n",
    "    surf = ax.plot_surface(x2, y2, z2, rstride=1, cstride=1, cmap=matplotlib.cm.coolwarm,\n",
    "        linewidth=0, antialiased=False)\n",
    "    fig.colorbar(surf)\n",
    "    \n",
    "    plt.show()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9a196e2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9523b923",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6794fe2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e62dbce",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66777dd2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6315344d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "744fdc6c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acc27b29",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "55266352",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80410b6b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7433b448",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96af8865",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6690323",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60da89c8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef51cb1e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b25b9eb6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc10db8b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a0215ec8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc047c12",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d01d3c9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}