ensisagent.py

import random
import math
import os.path

import numpy as np
import pandas as pd

from pysc2.agents import base_agent
from pysc2.lib import actions
from pysc2.lib import features

Max_Barracks = 7    #up to 7
Max_Supply_Depot = 6    #up to 6

_NO_OP = actions.FUNCTIONS.no_op.id
_SELECT_POINT = actions.FUNCTIONS.select_point.id
_BUILD_SUPPLY_DEPOT = actions.FUNCTIONS.Build_SupplyDepot_screen.id
_BUILD_BARRACKS = actions.FUNCTIONS.Build_Barracks_screen.id
_TRAIN_MARINE = actions.FUNCTIONS.Train_Marine_quick.id
_SELECT_ARMY = actions.FUNCTIONS.select_army.id
_ATTACK_MINIMAP = actions.FUNCTIONS.Attack_minimap.id
_HARVEST_GATHER = actions.FUNCTIONS.Harvest_Gather_screen.id
_BUILD_REFINERY = actions.FUNCTIONS.Build_Refinery_screen.id
_TRAIN_MARAUDER = actions.FUNCTIONS.Train_Marauder_quick.id
_BUILD_TECHLAB = actions.FUNCTIONS.Build_TechLab_screen.id    #Build_TechLab_quick seems not to work
_BUILD_TECHLABq = actions.FUNCTIONS.Build_TechLab_quick.id
_TRAIN_REAPER = actions.FUNCTIONS.Train_Reaper_quick.id
_TRAIN_SCV = actions.FUNCTIONS.Train_SCV_quick.id

_PLAYER_RELATIVE = features.SCREEN_FEATURES.player_relative.index
_UNIT_TYPE = features.SCREEN_FEATURES.unit_type.index
_PLAYER_ID = features.SCREEN_FEATURES.player_id.index

_PLAYER_SELF = 1
_PLAYER_HOSTILE = 4
_ARMY_SUPPLY = 5

_TERRAN_COMMANDCENTER = 18
_TERRAN_SCV = 45
_TERRAN_SUPPLY_DEPOT = 19
_TERRAN_BARRACKS = 21
_TERRAN_BARRACKSTECHLAB = 37
_TERRAN_BARRACKSREACTOR = 38
_NEUTRAL_MINERAL_FIELD = 341
_NEUTRAL_VESPENEGEYSER = 342 #_GEYSER = 343
_TERRAN_REFINERY = 20

_NOT_QUEUED = [0]
_QUEUED = [1]
_SELECT_ALL = [2]

DATA_FILE = 'sparse_agent_data'

ACTION_DO_NOTHING = 'donothing'
ACTION_BUILD_SUPPLY_DEPOT = 'buildsupplydepot'
ACTION_BUILD_BARRACKS = 'buildbarracks'
ACTION_BUILD_MARINE = 'buildmarine'
ACTION_ATTACK = 'attack'
ACTION_BUILD_REFINERY = 'buildrefinery'
ACTION_BUILD_MARAUDER = 'buildmarauder'
ACTION_BUILD_REAPER = 'buildreaper'
ACTION_BUILD_SCV = 'buildscv'

smart_actions = [
        ACTION_DO_NOTHING,
        ACTION_BUILD_SUPPLY_DEPOT,
        ACTION_BUILD_BARRACKS,
        ACTION_BUILD_MARINE,
		 ACTION_BUILD_REFINERY,
#		 ACTION_BUILD_MARAUDER,    Not working !
		 ACTION_BUILD_REAPER,
		 ACTION_BUILD_SCV,
]

for mm_x in range(0, 64):
	for mm_y in range(0, 64):
		if (mm_x + 1) % 16 == 0 and (mm_y + 1) % 16 == 0:
			smart_actions.append(ACTION_ATTACK + '_' + str(mm_x - 8) + '_' + str(mm_y - 8))

# python -m pysc2.bin.agent --map Simple64 --agent pysc2.agents.ensisagent.SparseAgent --agent_race T --max_agent_steps 0 --norender
#JB python -m pysc2.bin.agent --map Simple64 --agent pysc2.agents.SC2Agent.ensisagent.SparseAgent --agent_race T --max_agent_steps 0 --norender

# Stolen from https://github.com/MorvanZhou/Reinforcement-learning-with-tensorflow
class QLearningTable:
	def __init__(self, actions, learning_rate=0.1, reward_decay=0.9, e_greedy=0.85):    #initialy learning_rate=0.01, reward_decay=0.9, e_greedy=0.85
		self.actions = actions  # a list
		self.lr = learning_rate
		self.gamma = reward_decay
		self.epsilon = e_greedy
		self.q_table = pd.DataFrame(columns=self.actions, dtype=np.float64)

	def choose_action(self, observation):
		self.check_state_exist(observation)

		if np.random.uniform() < self.epsilon:
			# choose best action
			state_action = self.q_table.loc[observation, :]

			# some actions have the same value
			state_action = state_action.reindex(np.random.permutation(state_action.index))

			action = state_action.idxmax()
		else:
			# choose random action
			action = np.random.choice(self.actions)

		return action

	def learn(self, s, a, r, s_):
		self.check_state_exist(s_)
		self.check_state_exist(s)

		q_predict = self.q_table.loc[s, a]

		if s_ != 'terminal':
			q_target = r + self.gamma * self.q_table.loc[s_, :].max()
		else:
			q_target = r  # next state is terminal

		# update
		self.q_table.loc[s, a] += self.lr * (q_target - q_predict)

	def check_state_exist(self, state):
		if state not in self.q_table.index:
			# append new state to q table
			self.q_table = self.q_table.append(pd.Series([0] * len(self.actions), index=self.q_table.columns, name=state))

class SparseAgent(base_agent.BaseAgent):
	def __init__(self):
		super(SparseAgent, self).__init__()

		self.qlearn = QLearningTable(actions=list(range(len(smart_actions))))

		self.previous_action = None
		self.previous_state = None

		self.cc_y = None
		self.cc_x = None

		self.isTechlab = 0
		self.scv_count = 12

		self.move_number = 0

		if os.path.isfile(DATA_FILE + '.gz'):
			self.qlearn.q_table = pd.read_pickle(DATA_FILE + '.gz', compression='gzip')

	def transformDistance(self, x, x_distance, y, y_distance):
		if not self.base_top_left:
			return [x - x_distance, y - y_distance]

		return [x + x_distance, y + y_distance]

	def transformLocation(self, x, y):
		if not self.base_top_left:
			return [64 - x, 64 - y]

		return [x, y]

	def splitAction(self, action_id):
		smart_action = smart_actions[action_id]

		x = 0
		y = 0
		if '_' in smart_action:
			smart_action, x, y = smart_action.split('_')

		return (smart_action, x, y)

	def step(self, obs):
		super(SparseAgent, self).step(obs)

		if obs.last():
			reward = obs.reward

			if reward == 1:
				modreward = reward * 100000
			else:
				modreward = reward

			self.qlearn.learn(str(self.previous_state), self.previous_action, modreward, 'terminal')

			if reward == 1:
				reponse = 'VICTORY'
			if reward == 0:
				reponse = 'DRAW'
			if reward == -1:
				reponse = 'DEFEAT'

			file = open("ResultRecord.txt","a")
			file.write(str(reponse) + '\n')
			file.close()

			self.qlearn.q_table.to_pickle(DATA_FILE + '.gz', 'gzip')

			self.previous_action = None
			self.previous_state = None

			self.move_number = 0

			return actions.FunctionCall(_NO_OP, [])

		unit_type = obs.observation['feature_screen'][_UNIT_TYPE]

		if obs.first():
			player_y, player_x = (obs.observation['feature_minimap'][_PLAYER_RELATIVE] == _PLAYER_SELF).nonzero()
			self.base_top_left = 1 if player_y.any() and player_y.mean() <= 31 else 0

			self.cc_y, self.cc_x = (unit_type == _TERRAN_COMMANDCENTER).nonzero()

			self.isTechlab = 0
			self.scv_count = 12

		cc_y, cc_x = (unit_type == _TERRAN_COMMANDCENTER).nonzero()
		cc_count = 1 if cc_y.any() else 0

		depot_y, depot_x = (unit_type == _TERRAN_SUPPLY_DEPOT).nonzero()
		supply_depot_count = int(round(len(depot_y) / 69))

		barracks_y, barracks_x = (unit_type == _TERRAN_BARRACKS).nonzero()
		barracks_count = int(round(len(barracks_y) / 137))

		refinery_y, refinery_x = (unit_type == _TERRAN_REFINERY).nonzero()
		refinery_count = int(round(len(refinery_y) / 137))

		if self.move_number == 0:
			self.move_number += 1

			current_state = np.zeros(38)
			current_state[0] = cc_count
			current_state[1] = supply_depot_count
			current_state[2] = barracks_count
			current_state[3] = obs.observation['player'][_ARMY_SUPPLY]
			current_state[4] = refinery_count
			current_state[5] = self.scv_count

			hot_squares = np.zeros(16)
			enemy_y, enemy_x = (obs.observation['feature_minimap'][_PLAYER_RELATIVE] == _PLAYER_HOSTILE).nonzero()
			for i in range(0, len(enemy_y)):
				y = int(math.ceil((enemy_y[i] + 1) / 16))
				x = int(math.ceil((enemy_x[i] + 1) / 16))

				hot_squares[((y - 1) * 4) + (x - 1)] = 1

			if not self.base_top_left:
				hot_squares = hot_squares[::-1]

			for i in range(0, 16):
				current_state[i + 6] = hot_squares[i]

			raid_squares = np.zeros(16)
			ally_y, ally_x = (obs.observation['feature_minimap'][_PLAYER_RELATIVE] == _PLAYER_SELF).nonzero()
			for i in range(0, len(ally_y)):
				y = int(math.ceil((ally_y[i] + 1) / 16))
				x = int(math.ceil((ally_x[i] + 1) / 16))

				raid_squares[((y - 1) * 4) + (x - 1)] = 1

			if not self.base_top_left:
				raid_squares = raid_squares[::-1]

			for i in range(0, 16):
				current_state[i + 22] = raid_squares[i]

			if self.previous_action is not None:
				self.qlearn.learn(str(self.previous_state), self.previous_action, 0, str(current_state))

			rl_action = self.qlearn.choose_action(str(current_state))

			self.previous_state = current_state
			self.previous_action = rl_action

			smart_action, x, y = self.splitAction(self.previous_action)

			if smart_action == ACTION_BUILD_BARRACKS or smart_action == ACTION_BUILD_SUPPLY_DEPOT or smart_action == ACTION_BUILD_REFINERY:
				unit_y, unit_x = (unit_type == _TERRAN_SCV).nonzero()

				if unit_y.any():
					i = random.randint(0, len(unit_y) - 1)
					target = [unit_x[i], unit_y[i]]

					return actions.FunctionCall(_SELECT_POINT, [_NOT_QUEUED, target])

			elif smart_action == ACTION_BUILD_MARINE or smart_action == ACTION_BUILD_MARAUDER or smart_action == ACTION_BUILD_REAPER:
				if barracks_y.any():
					i = random.randint(0, len(barracks_y) - 1)
					target = [barracks_x[i], barracks_y[i]]

					return actions.FunctionCall(_SELECT_POINT, [_SELECT_ALL, target])

			elif smart_action == ACTION_ATTACK:
				if _SELECT_ARMY in obs.observation['available_actions']:
					return actions.FunctionCall(_SELECT_ARMY, [_NOT_QUEUED])

			elif smart_action == ACTION_BUILD_SCV:
				if self.cc_y.any():
					return actions.FunctionCall(_SELECT_POINT, [_NOT_QUEUED, [round(self.cc_x.mean()),round(self.cc_y.mean())]])

		elif self.move_number == 1:
			self.move_number += 1

			smart_action, x, y = self.splitAction(self.previous_action)

			if smart_action == ACTION_BUILD_REFINERY:
				if refinery_count < 2 and _BUILD_REFINERY in obs.observation['available_actions'] and barracks_count == Max_Barracks and supply_depot_count == Max_Supply_Depot:
					if self.cc_y.any():
						if refinery_count == 0:
							unit_x, unit_y = (unit_type == _NEUTRAL_VESPENEGEYSER).nonzero()
							if unit_y.any():
								#i = random.randint(0, len(unit_y) - 1)
								i = int(math.ceil((len(unit_y)/4)))
								t_y,t_x = unit_y[i],unit_x[i]
							target = [t_y,t_x]
						elif refinery_count == 1:
							unit_x, unit_y = (unit_type == _NEUTRAL_VESPENEGEYSER).nonzero()
							if unit_y.any():
								#i = random.randint(0, len(unit_y) - 1)
								i = int(round(len(unit_y)/4))    #round originaly math.ceil
								t_y,t_x = unit_y[3*i-1],unit_x[3*i-1]
							target = [t_y,t_x]

						return actions.FunctionCall(_BUILD_REFINERY, [_NOT_QUEUED, target])

			if smart_action == ACTION_BUILD_SUPPLY_DEPOT:
				if supply_depot_count < Max_Supply_Depot and _BUILD_SUPPLY_DEPOT in obs.observation['available_actions']:
					if self.cc_y.any():
						if supply_depot_count == 0:
							target = self.transformDistance(round(self.cc_x.mean()), -35, round(self.cc_y.mean()), 0)
						elif supply_depot_count == 1:
							target = self.transformDistance(round(self.cc_x.mean()), -25, round(self.cc_y.mean()), -25)
						elif supply_depot_count == 2:
							target = self.transformDistance(round(self.cc_x.mean()), -15, round(self.cc_y.mean()), -35)
						elif supply_depot_count == 3:
							target = self.transformDistance(round(self.cc_x.mean()), -30, round(self.cc_y.mean()), -8)
						elif supply_depot_count == 4:
							target = self.transformDistance(round(self.cc_x.mean()), -35, round(self.cc_y.mean()), -16)
						elif supply_depot_count == 5:
							target = self.transformDistance(round(self.cc_x.mean()), -5, round(self.cc_y.mean()), -30)

						return actions.FunctionCall(_BUILD_SUPPLY_DEPOT, [_NOT_QUEUED, target])

			elif smart_action == ACTION_BUILD_BARRACKS:
				if barracks_count < Max_Barracks and _BUILD_BARRACKS in obs.observation['available_actions']:
					if self.cc_y.any():
						if  barracks_count == 0:
							target = self.transformDistance(round(self.cc_x.mean()), 15, round(self.cc_y.mean()), -12)    #initialy 15,-9
						elif  barracks_count == 1:
							target = self.transformDistance(round(self.cc_x.mean()), 15, round(self.cc_y.mean()), 12)
						elif barracks_count == 2:
							target = self.transformDistance(round(self.cc_x.mean()), 15, round(self.cc_y.mean()), 25)    #y != 30    x is the absciss // x,y=15,25 is the  out angle
						elif barracks_count == 3:
							target = self.transformDistance(round(self.cc_x.mean()), 15, round(self.cc_y.mean()), 0)
						elif barracks_count == 4:
							target = self.transformDistance(round(self.cc_x.mean()), 28, round(self.cc_y.mean()), -12)
						elif  barracks_count == 5:
							target = self.transformDistance(round(self.cc_x.mean()), 28, round(self.cc_y.mean()), 12)
						elif barracks_count == 6:
							target = self.transformDistance(round(self.cc_x.mean()), 28, round(self.cc_y.mean()), 0)

					return actions.FunctionCall(_BUILD_BARRACKS, [_NOT_QUEUED, target])

			elif smart_action == ACTION_BUILD_MARINE:
				if _TRAIN_MARINE in obs.observation['available_actions']:
					return actions.FunctionCall(_TRAIN_MARINE, [_QUEUED])

			elif smart_action == ACTION_BUILD_REAPER:
				if _TRAIN_REAPER in obs.observation['available_actions']:
					return actions.FunctionCall(_TRAIN_REAPER, [_QUEUED])

			elif smart_action == ACTION_BUILD_MARAUDER:
				if _BUILD_TECHLAB in obs.observation['available_actions'] and self.isTechlab < 1:
					target = self.transformDistance(round(self.cc_x.mean()), -35, round(self.cc_y.mean()), 0)
					#target[0] += random.randint(-5,5)
					#target[1] += random.randint(-5,5)
					print("on lance la recherche TECHLAB en ",target)
					self.isTechlab += 1
					print ("self.isTechlab = ", self.isTechlab)
					return actions.FunctionCall(_BUILD_TECHLAB, [_NOT_QUEUED, target])
					#return actions.FunctionCall(_BUILD_TECHLABq, [_NOT_QUEUED])
				elif _TRAIN_MARAUDER in obs.observation['available_actions']:
					print("l'ordre est disponible !")
					return actions.FunctionCall(_TRAIN_MARAUDER, [_QUEUED])

			elif smart_action == ACTION_ATTACK:
				do_it = True

				if len(obs.observation['single_select']) > 0 and obs.observation['single_select'][0][0] == _TERRAN_SCV:
					do_it = False

				if len(obs.observation['multi_select']) > 0 and obs.observation['multi_select'][0][0] == _TERRAN_SCV:
					do_it = False

				if do_it and _ATTACK_MINIMAP in obs.observation["available_actions"]:
					x_offset = random.randint(-1, 1)
					y_offset = random.randint(-1, 1)

					return actions.FunctionCall(_ATTACK_MINIMAP, [_NOT_QUEUED, self.transformLocation(int(x) + (x_offset * 4), int(y) + (y_offset * 4))])

			elif smart_action == ACTION_BUILD_SCV:
				if self.scv_count < 20 and _TRAIN_SCV in obs.observation['available_actions']:
					self.scv_count += 1
					return actions.FunctionCall(_TRAIN_SCV, [_QUEUED])

		elif self.move_number == 2:
			self.move_number = 0

			smart_action, x, y = self.splitAction(self.previous_action)

			if smart_action == ACTION_BUILD_BARRACKS or smart_action == ACTION_BUILD_SUPPLY_DEPOT:
				if _HARVEST_GATHER in obs.observation['available_actions']:
					unit_y, unit_x = (unit_type == _NEUTRAL_MINERAL_FIELD).nonzero()

					if unit_y.any():
						i = random.randint(0, len(unit_y) - 1)

						m_x = unit_x[i]
						m_y = unit_y[i]

						target = [int(m_x), int(m_y)]
						if barracks_count != Max_Barracks or supply_depot_count != Max_Supply_Depot:
							return actions.FunctionCall(_HARVEST_GATHER, [_QUEUED, target])

						#return actions.FunctionCall(_HARVEST_GATHER, [_QUEUED, target])
						return actions.FunctionCall(_NO_OP, [])

			elif smart_action == ACTION_BUILD_REFINERY:
				return actions.FunctionCall(_NO_OP, [])

		return actions.FunctionCall(_NO_OP, [])