diff --git a/README.md b/README.md
index ba780652..27e5ea0d 100755
--- a/README.md
+++ b/README.md
@@ -1,20 +1,26 @@
+
+
+[](https://www.repostatus.org/#active)
+[](https://github.com/LucasAlegre/sumo-rl/blob/master/LICENSE)
+
+
# SUMO-RL
-SUMO-RL provides a simple interface to instantiate Reinforcement Learning environments with [SUMO v.1.5.0](https://github.com/eclipse/sumo) for Traffic Signal Control.
+SUMO-RL provides a simple interface to instantiate Reinforcement Learning environments with [SUMO](https://github.com/eclipse/sumo) for Traffic Signal Control.
The main class [SumoEnvironment](https://github.com/LucasAlegre/sumo-rl/blob/master/environment/env.py) inherits [MultiAgentEnv](https://github.com/ray-project/ray/blob/master/python/ray/rllib/env/multi_agent_env.py) from [RLlib](https://github.com/ray-project/ray/tree/master/python/ray/rllib).
If instantiated with parameter 'single-agent=True', it behaves like a regular [Gym Env](https://github.com/openai/gym/blob/master/gym/core.py) from [OpenAI](https://github.com/openai).
[TrafficSignal](https://github.com/LucasAlegre/sumo-rl/blob/master/environment/traffic_signal.py) is responsible for retrieving information and actuating on traffic lights using [TraCI](https://sumo.dlr.de/wiki/TraCI) API.
Goals of this repository:
-- Provide a simple interface to work with Reinforcement Learning for Traffic Signal Control using SUMO.
-- Support Multiagent RL.
-- Compatibility with Gym Env and popular RL libraries like openAI baselines and RLlib.
-- Easy customisation: state and reward definitions are easily modifiable.
+- Provide a simple interface to work with Reinforcement Learning for Traffic Signal Control using SUMO
+- Support Multiagent RL
+- Compatibility with Gym Env and popular RL libraries like openAI baselines and RLlib
+- Easy customisation: state and reward definitions are easily modifiable
## Install
-### To install SUMO v1.5.0:
+### Install SUMO latest version:
```
sudo add-apt-repository ppa:sumo/stable
@@ -27,9 +33,18 @@ echo 'export SUMO_HOME="/usr/share/sumo"' >> ~/.bashrc
source ~/.bashrc
```
-### To install sumo_rl package:
+### Install SUMO-RL
+
+Stable release version is available through pip
+```
+pip install sumo-rl
+```
+
+Alternatively you can install using the latest (unreleased) version
```
-pip3 install -e .
+git clone https://github.com/LucasAlegre/sumo-rl
+cd sumo-rl
+pip install -e .
```
## Examples
@@ -46,18 +61,18 @@ python3 experiments/ql_single-intersection.py
python3 experiments/a3c_4x4grid.py
```
-### [stable-baselines A2C](https://stable-baselines.readthedocs.io/en/master/modules/a2c.html) in a 2-way single intersection:
+### [stable-baselines3 DQN](https://github.com/DLR-RM/stable-baselines3/blob/master/stable_baselines3/dqn/dqn.py) in a 2-way single intersection:
```
-python3 experiments/a2c_2way-single-intersection.py
+python3 experiments/dqn_2way-single-intersection.py
```
-### To plot results:
+### Plotting results:
```
python3 outputs/plot.py -f outputs/2way-single-intersection/a3c
```
-## Cite
+## Citation
If you use this repository in your research, please cite:
```
@misc{sumorl,
diff --git a/experiments/a2c_2way-single-intersection.py b/experiments/a2c_2way-single-intersection.py
index 0c39f51a..1e0717b3 100755
--- a/experiments/a2c_2way-single-intersection.py
+++ b/experiments/a2c_2way-single-intersection.py
@@ -19,29 +19,21 @@
from stable_baselines.common.vec_env import SubprocVecEnv
from stable_baselines import A2C
-write_route_file('nets/2way-single-intersection/single-intersection-gen.rou.xml', 400000, 100000)
-# multiprocess environment
-n_cpu = 2
-env = SubprocVecEnv([lambda: SumoEnvironment(net_file='nets/2way-single-intersection/single-intersection.net.xml',
- route_file='nets/2way-single-intersection/single-intersection-gen.rou.xml',
- out_csv_name='outputs/2way-single-intersection/a2c-contexts-5s-vmvm-400k',
- single_agent=True,
- use_gui=True,
- num_seconds=400000,
- min_green=5,
- time_to_load_vehicles=120,
- max_depart_delay=0,
- phases=[
- traci.trafficlight.Phase(32, "GGrrrrGGrrrr"),
- traci.trafficlight.Phase(2, "yyrrrryyrrrr"),
- traci.trafficlight.Phase(32, "rrGrrrrrGrrr"),
- traci.trafficlight.Phase(2, "rryrrrrryrrr"),
- traci.trafficlight.Phase(32, "rrrGGrrrrGGr"),
- traci.trafficlight.Phase(2, "rrryyrrrryyr"),
- traci.trafficlight.Phase(32, "rrrrrGrrrrrG"),
- traci.trafficlight.Phase(2, "rrrrryrrrrry")
- ]) for i in range(n_cpu)])
+if __name__ == '__main__':
-model = A2C(MlpPolicy, env, verbose=1, learning_rate=0.0001, lr_schedule='constant')
-model.learn(total_timesteps=1000000)
+ write_route_file('nets/2way-single-intersection/single-intersection-gen.rou.xml', 400000, 100000)
+
+ # multiprocess environment
+ n_cpu = 1
+ env = SubprocVecEnv([lambda: SumoEnvironment(net_file='nets/2way-single-intersection/single-intersection.net.xml',
+ route_file='nets/2way-single-intersection/single-intersection-gen.rou.xml',
+ out_csv_name='outputs/2way-single-intersection/a2c',
+ single_agent=True,
+ use_gui=False,
+ num_seconds=100000,
+ min_green=5,
+ max_depart_delay=0) for _ in range(n_cpu)])
+
+ model = A2C(MlpPolicy, env, verbose=1, learning_rate=0.001, lr_schedule='constant')
+ model.learn(total_timesteps=100000)
diff --git a/experiments/a3c_2way-single-intersection.py b/experiments/a3c_2way-single-intersection.py
deleted file mode 100755
index 46e05950..00000000
--- a/experiments/a3c_2way-single-intersection.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import argparse
-import os
-import sys
-if 'SUMO_HOME' in os.environ:
- tools = os.path.join(os.environ['SUMO_HOME'], 'tools')
- sys.path.append(tools)
-else:
- sys.exit("Please declare the environment variable 'SUMO_HOME'")
-import pandas as pd
-import ray
-from ray.rllib.agents.a3c.a3c import A3CTrainer
-from ray.rllib.agents.a3c.a3c_tf_policy import A3CTFPolicy
-from ray.tune.registry import register_env
-from ray.tune.logger import pretty_print
-from gym import spaces
-import numpy as np
-from sumo_rl.environment.env import SumoEnvironment
-import traci
-
-def policy_mapping(id):
- return '0'
-
-if __name__ == '__main__':
- ray.init()
-
- register_env("2way-single-intersection", lambda _: SumoEnvironment(net_file='nets/2way-single-intersection/single-intersection.net.xml',
- route_file='nets/2way-single-intersection/single-intersection-gen.rou.xml',
- out_csv_name='outputs/2way-single-intersection/a3c-contexts',
- use_gui=False,
- num_seconds=100000,
- time_to_load_vehicles=120,
- max_depart_delay=0,
- phases=[
- traci.trafficlight.Phase(32, "GGrrrrGGrrrr"),
- traci.trafficlight.Phase(2, "yyrrrryyrrrr"),
- traci.trafficlight.Phase(32, "rrGrrrrrGrrr"),
- traci.trafficlight.Phase(2, "rryrrrrryrrr"),
- traci.trafficlight.Phase(32, "rrrGGrrrrGGr"),
- traci.trafficlight.Phase(2, "rrryyrrrryyr"),
- traci.trafficlight.Phase(32, "rrrrrGrrrrrG"),
- traci.trafficlight.Phase(2, "rrrrryrrrrry")
- ]))
-
- trainer = A3CTrainer(env="2way-single-intersection", config={
- "multiagent": {
- "policy_graphs": {
- '0': (A3CTFPolicy, spaces.Box(low=np.zeros(20), high=np.ones(20)), spaces.Discrete(4), {})
- },
- "policy_mapping_fn": policy_mapping # Traffic lights are always controlled by this policy
- },
- "lr": 0.0001,
- })
-
- while True:
- result = trainer.train()
- print(pretty_print(result))
\ No newline at end of file
diff --git a/experiments/a3c_2x2grid.py b/experiments/a3c_2x2grid.py
deleted file mode 100755
index ad0c3149..00000000
--- a/experiments/a3c_2x2grid.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import argparse
-import os
-import sys
-if 'SUMO_HOME' in os.environ:
- tools = os.path.join(os.environ['SUMO_HOME'], 'tools')
- sys.path.append(tools)
-else:
- sys.exit("Please declare the environment variable 'SUMO_HOME'")
-import pandas as pd
-import ray
-from ray.rllib.agents.a3c.a3c import A3CTrainer
-from ray.rllib.agents.a3c.a3c_tf_policy import A3CTFPolicy
-from ray.tune.registry import register_env
-from ray.tune.logger import pretty_print
-from gym import spaces
-import numpy as np
-from sumo_rl.environment.env import SumoEnvironment
-import traci
-
-def policy_mapping(id):
- return '0'
-
-if __name__ == '__main__':
- ray.init()
-
- register_env("2x2grid", lambda _: SumoEnvironment(net_file='nets/2x2grid/2x2.net.xml',
- route_file='nets/2x2grid/2x2.rou.xml',
- out_csv_name='outputs/2x2grid/a3c-contexts',
- use_gui=True,
- num_seconds=100000,
- time_to_load_vehicles=120,
- max_depart_delay=0,
- phases=[
- traci.trafficlight.Phase(32, "GGrrrrGGrrrr"),
- traci.trafficlight.Phase(2, "yyrrrryyrrrr"),
- traci.trafficlight.Phase(32, "rrGrrrrrGrrr"),
- traci.trafficlight.Phase(2, "rryrrrrryrrr"),
- traci.trafficlight.Phase(32, "rrrGGrrrrGGr"),
- traci.trafficlight.Phase(2, "rrryyrrrryyr"),
- traci.trafficlight.Phase(32, "rrrrrGrrrrrG"),
- traci.trafficlight.Phase(2, "rrrrryrrrrry")
- ]))
-
- trainer = A3CTrainer(env="2x2grid", config={
- "multiagent": {
- "policy_graphs": {
- '0': (A3CTFPolicy, spaces.Box(low=np.zeros(20), high=np.ones(20)), spaces.Discrete(4), {})
- },
- "policy_mapping_fn": policy_mapping # Traffic lights are always controlled by this policy
- },
- "lr": 0.0001,
- })
-
- while True:
- result = trainer.train()
- print(pretty_print(result))
\ No newline at end of file
diff --git a/experiments/a3c_4x4grid.py b/experiments/a3c_4x4grid.py
index b8d33382..d1357dd3 100755
--- a/experiments/a3c_4x4grid.py
+++ b/experiments/a3c_4x4grid.py
@@ -22,26 +22,20 @@
register_env("4x4grid", lambda _: SumoEnvironment(net_file='nets/4x4-Lucas/4x4.net.xml',
route_file='nets/4x4-Lucas/4x4c1c2c1c2.rou.xml',
- out_csv_name='outputs/4x4grid/a3c-4x4grid',
+ out_csv_name='outputs/4x4grid/a3c',
use_gui=False,
num_seconds=80000,
- time_to_load_vehicles=120,
- max_depart_delay=0,
- phases=[
- traci.trafficlight.Phase(35, "GGGrrr"), # north-south
- traci.trafficlight.Phase(2, "yyyrrr"),
- traci.trafficlight.Phase(35, "rrrGGG"), # west-east
- traci.trafficlight.Phase(2, "rrryyy")
- ]))
+ max_depart_delay=0))
trainer = A3CTrainer(env="4x4grid", config={
"multiagent": {
- "policy_graphs": {
+ "policies": {
'0': (A3CTFPolicy, spaces.Box(low=np.zeros(10), high=np.ones(10)), spaces.Discrete(2), {})
},
- "policy_mapping_fn": lambda id: '0' # Traffic lights are always controlled by this policy
+ "policy_mapping_fn": (lambda id: '0') # Traffic lights are always controlled by this policy
},
- "lr": 0.0001,
+ "lr": 0.001,
+ "no_done_at_end": True
})
while True:
print(trainer.train()) # distributed training step
\ No newline at end of file
diff --git a/experiments/dqn_2way-single-intersection.py b/experiments/dqn_2way-single-intersection.py
index 10cac6e0..04d2be6f 100755
--- a/experiments/dqn_2way-single-intersection.py
+++ b/experiments/dqn_2way-single-intersection.py
@@ -1,7 +1,7 @@
import gym
import numpy as np
-from stable_baselines.deepq import DQN, MlpPolicy
+from stable_baselines3.dqn.dqn import DQN
import argparse
import os
@@ -22,29 +22,21 @@
env = SumoEnvironment(net_file='nets/2way-single-intersection/single-intersection.net.xml',
route_file='nets/2way-single-intersection/single-intersection-vhvh.rou.xml',
- out_csv_name='outputs/2way-single-intersection/dqn-vhvh2-stable-mlp-bs',
+ out_csv_name='outputs/2way-single-intersection/dqn',
single_agent=True,
- use_gui=True,
+ use_gui=False,
num_seconds=100000,
- time_to_load_vehicles=120,
- max_depart_delay=0,
- phases=[
- traci.trafficlight.Phase(32, "GGrrrrGGrrrr"),
- traci.trafficlight.Phase(2, "yyrrrryyrrrr"),
- traci.trafficlight.Phase(32, "rrGrrrrrGrrr"),
- traci.trafficlight.Phase(2, "rryrrrrryrrr"),
- traci.trafficlight.Phase(32, "rrrGGrrrrGGr"),
- traci.trafficlight.Phase(2, "rrryyrrrryyr"),
- traci.trafficlight.Phase(32, "rrrrrGrrrrrG"),
- traci.trafficlight.Phase(2, "rrrrryrrrrry")
- ])
+ max_depart_delay=0)
model = DQN(
env=env,
- policy=MlpPolicy,
- learning_rate=1e-3,
- buffer_size=50000,
- exploration_fraction=0.1,
- exploration_final_eps=0.02
+ policy="MlpPolicy",
+ learning_rate=0.01,
+ learning_starts=0,
+ train_freq=1,
+ target_update_interval=100,
+ exploration_initial_eps=0.05,
+ exploration_final_eps=0.01,
+ verbose=1
)
model.learn(total_timesteps=100000)
\ No newline at end of file
diff --git a/experiments/dqn_big-intersection.py b/experiments/dqn_big-intersection.py
index df1013b2..ebeb3769 100644
--- a/experiments/dqn_big-intersection.py
+++ b/experiments/dqn_big-intersection.py
@@ -26,17 +26,7 @@
yellow_time=4,
min_green=5,
max_green=60,
- max_depart_delay=0,
- time_to_load_vehicles=0,
- phases=[
- traci.trafficlight.Phase(30, "GGGGrrrrrrGGGGrrrrrr"),
- traci.trafficlight.Phase(4, "yyyyrrrrrryyyyrrrrrr"),
- traci.trafficlight.Phase(15, "rrrrGrrrrrrrrrGrrrrr"),
- traci.trafficlight.Phase(4, "rrrryrrrrrrrrryrrrrr"),
- traci.trafficlight.Phase(30, "rrrrrGGGGrrrrrrGGGGr"),
- traci.trafficlight.Phase(4, "rrrrryyyyrrrrrryyyyr"),
- traci.trafficlight.Phase(15, "rrrrrrrrrGrrrrrrrrrG"),
- traci.trafficlight.Phase(4, "rrrrrrrrryrrrrrrrrry")])
+ max_depart_delay=0)
model = DQN(
env=env,
diff --git a/experiments/ql_2way-single-intersection.py b/experiments/ql_2way-single-intersection.py
index 7d1a0166..6a083b06 100755
--- a/experiments/ql_2way-single-intersection.py
+++ b/experiments/ql_2way-single-intersection.py
@@ -32,7 +32,6 @@
prs.add_argument("-fixed", action="store_true", default=False, help="Run with fixed timing traffic signals.\n")
prs.add_argument("-s", dest="seconds", type=int, default=100000, required=False, help="Number of simulation seconds.\n")
prs.add_argument("-r", dest="reward", type=str, default='wait', required=False, help="Reward function: [-r queue] for average queue reward or [-r wait] for waiting time reward.\n")
- prs.add_argument("-v", action="store_true", default=False, help="Print experience tuple.\n")
prs.add_argument("-runs", dest="runs", type=int, default=1, help="Number of runs.\n")
args = prs.parse_args()
experiment_time = str(datetime.now()).split('.')[0]
@@ -45,26 +44,11 @@
num_seconds=args.seconds,
min_green=args.min_green,
max_green=args.max_green,
- max_depart_delay=0,
- time_to_load_vehicles=120,
- phases=[
- traci.trafficlight.Phase(32, "GGrrrrGGrrrr"),
- traci.trafficlight.Phase(2, "yyrrrryyrrrr"),
- traci.trafficlight.Phase(32, "rrGrrrrrGrrr"),
- traci.trafficlight.Phase(2, "rryrrrrryrrr"),
- traci.trafficlight.Phase(32, "rrrGGrrrrGGr"),
- traci.trafficlight.Phase(2, "rrryyrrrryyr"),
- traci.trafficlight.Phase(32, "rrrrrGrrrrrG"),
- traci.trafficlight.Phase(2, "rrrrryrrrrry")
- ])
- if args.reward == 'queue':
- env._compute_rewards = env._queue_average_reward
- else:
- env._compute_rewards = env._waiting_time_reward
+ max_depart_delay=0)
for run in range(1, args.runs+1):
initial_states = env.reset()
- ql_agents = {ts: QLAgent(starting_state=env.encode(initial_states[ts]),
+ ql_agents = {ts: QLAgent(starting_state=env.encode(initial_states[ts], ts),
state_space=env.observation_space,
action_space=env.action_space,
alpha=args.alpha,
@@ -82,11 +66,8 @@
s, r, done, _ = env.step(action=actions)
- if args.v:
- print('s=', env.radix_decode(ql_agents['t'].state), 'a=', actions['t'], 's\'=', env.radix_encode(s['t']), 'r=', r['t'])
-
for agent_id in ql_agents.keys():
- ql_agents[agent_id].learn(next_state=env.encode(s[agent_id]), reward=r[agent_id])
+ ql_agents[agent_id].learn(next_state=env.encode(s[agent_id], agent_id), reward=r[agent_id])
env.save_csv(out_csv, run)
env.close()
diff --git a/experiments/ql_4x4grid.py b/experiments/ql_4x4grid.py
index 5198c83d..b5269d3e 100755
--- a/experiments/ql_4x4grid.py
+++ b/experiments/ql_4x4grid.py
@@ -23,21 +23,14 @@
runs = 1
env = SumoEnvironment(net_file='nets/4x4-Lucas/4x4.net.xml',
- route_file='nets/4x4-Lucas/4x4c2.rou.xml',
+ route_file='nets/4x4-Lucas/4x4c1c2c1c2.rou.xml',
use_gui=True,
- num_seconds=40000,
- time_to_load_vehicles=300,
- max_depart_delay=0,
- phases=[
- traci.trafficlight.Phase(35, "GGGrrr"), # north-south
- traci.trafficlight.Phase(2, "yyyrrr"),
- traci.trafficlight.Phase(35, "rrrGGG"), # west-east
- traci.trafficlight.Phase(2, "rrryyy")
- ])
+ num_seconds=80000,
+ max_depart_delay=0)
for run in range(1, runs+1):
initial_states = env.reset()
- ql_agents = {ts: QLAgent(starting_state=env.encode(initial_states[ts]),
+ ql_agents = {ts: QLAgent(starting_state=env.encode(initial_states[ts], ts),
state_space=env.observation_space,
action_space=env.action_space,
alpha=alpha,
@@ -49,13 +42,10 @@
actions = {ts: ql_agents[ts].act() for ts in ql_agents.keys()}
s, r, done, info = env.step(action=actions)
- infos.append(info)
- for agent_id in ql_agents.keys():
- ql_agents[agent_id].learn(next_state=env.encode(s[agent_id]), reward=r[agent_id])
+ for agent_id in s.keys():
+ ql_agents[agent_id].learn(next_state=env.encode(s[agent_id], agent_id), reward=r[agent_id])
env.close()
- df = pd.DataFrame(infos)
- df.to_csv('outputs/4x4grid/c2_alpha{}_gamma{}_decay{}_run{}.csv'.format(alpha, gamma, decay, run), index=False)
diff --git a/experiments/ql_single-intersection.py b/experiments/ql_single-intersection.py
index c54cdb19..392a96c5 100755
--- a/experiments/ql_single-intersection.py
+++ b/experiments/ql_single-intersection.py
@@ -33,7 +33,6 @@
prs.add_argument("-ns", dest="ns", type=int, default=42, required=False, help="Fixed green time for NS.\n")
prs.add_argument("-we", dest="we", type=int, default=42, required=False, help="Fixed green time for WE.\n")
prs.add_argument("-s", dest="seconds", type=int, default=100000, required=False, help="Number of simulation seconds.\n")
- prs.add_argument("-r", dest="reward", type=str, default='wait', required=False, help="Reward function: [-r queue] for average queue reward or [-r wait] for waiting time reward.\n")
prs.add_argument("-v", action="store_true", default=False, help="Print experience tuple.\n")
prs.add_argument("-runs", dest="runs", type=int, default=1, help="Number of runs.\n")
args = prs.parse_args()
@@ -47,21 +46,11 @@
num_seconds=args.seconds,
min_green=args.min_green,
max_green=args.max_green,
- max_depart_delay=0,
- phases=[
- traci.trafficlight.Phase(args.ns, "GGrr"), # north-south
- traci.trafficlight.Phase(2, "yyrr"),
- traci.trafficlight.Phase(args.we, "rrGG"), # west-east
- traci.trafficlight.Phase(2, "rryy")
- ])
- if args.reward == 'queue':
- env._compute_rewards = env._queue_average_reward
- else:
- env._compute_rewards = env._waiting_time_reward
+ max_depart_delay=0)
for run in range(1, args.runs+1):
initial_states = env.reset()
- ql_agents = {ts: QLAgent(starting_state=env.encode(initial_states[ts]),
+ ql_agents = {ts: QLAgent(starting_state=env.encode(initial_states[ts], ts),
state_space=env.observation_space,
action_space=env.action_space,
alpha=args.alpha,
@@ -79,11 +68,8 @@
s, r, done, _ = env.step(action=actions)
- if args.v:
- print('s=', env.radix_decode(ql_agents['t'].state), 'a=', actions['t'], 's\'=', env.radix_decode(env.encode(s['t'])), 'r=', r['t'])
-
for agent_id in ql_agents.keys():
- ql_agents[agent_id].learn(next_state=env.encode(s[agent_id]), reward=r[agent_id])
+ ql_agents[agent_id].learn(next_state=env.encode(s[agent_id], agent_id), reward=r[agent_id])
env.save_csv(out_csv, run)
env.close()
diff --git a/experiments/sarsa_2way-single-intersection.py b/experiments/sarsa_2way-single-intersection.py
index 3fd1b0b0..d9e25222 100644
--- a/experiments/sarsa_2way-single-intersection.py
+++ b/experiments/sarsa_2way-single-intersection.py
@@ -13,7 +13,7 @@
import traci
from sumo_rl.util.gen_route import write_route_file
from sumo_rl.environment.env import SumoEnvironment
-from sumo_rl.agents.sarsa_lambda import TrueOnlineSarsaLambda
+from linear_rl.true_online_sarsa import TrueOnlineSarsaLambda
if __name__ == '__main__':
@@ -21,9 +21,9 @@
prs = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="""SarsaLambda Single-Intersection""")
prs.add_argument("-route", dest="route", type=str, default='nets/2way-single-intersection/single-intersection-gen.rou.xml', help="Route definition xml file.\n")
- prs.add_argument("-a", dest="alpha", type=float, default=0.0001, required=False, help="Alpha learning rate.\n")
- prs.add_argument("-g", dest="gamma", type=float, default=0.99, required=False, help="Gamma discount rate.\n")
- prs.add_argument("-e", dest="epsilon", type=float, default=0.05, required=False, help="Epsilon.\n")
+ prs.add_argument("-a", dest="alpha", type=float, default=0.00001, required=False, help="Alpha learning rate.\n")
+ prs.add_argument("-g", dest="gamma", type=float, default=0.95, required=False, help="Gamma discount rate.\n")
+ prs.add_argument("-e", dest="epsilon", type=float, default=0.01, required=False, help="Epsilon.\n")
prs.add_argument("-mingreen", dest="min_green", type=int, default=5, required=False, help="Minimum green time.\n")
prs.add_argument("-maxgreen", dest="max_green", type=int, default=50, required=False, help="Maximum green time.\n")
prs.add_argument("-gui", action="store_true", default=False, help="Run with visualization on SUMO.\n")
@@ -31,11 +31,10 @@
prs.add_argument("-s", dest="seconds", type=int, default=400000, required=False, help="Number of simulation seconds.\n")
prs.add_argument("-runs", dest="runs", type=int, default=1, help="Number of runs.\n")
args = prs.parse_args()
- experiment_time = str(datetime.now()).split('.')[0]
- out_csv = 'outputs/2way-single-intersection/sarsa_lambda'
- write_route_file('nets/2way-single-intersection/single-intersection-gen.rou.xml', 400000, 100000)
+ out_csv = 'outputs/2way-single-intersection/sarsa_lambdavai'
+ write_route_file('nets/2way-single-intersection/single-intersection-gen.rou.xml', 400000, 100000)
env = SumoEnvironment(net_file='nets/2way-single-intersection/single-intersection.net.xml',
single_agent=True,
route_file=args.route,
@@ -44,22 +43,11 @@
num_seconds=args.seconds,
min_green=args.min_green,
max_green=args.max_green,
- max_depart_delay=0,
- time_to_load_vehicles=120,
- phases=[
- traci.trafficlight.Phase(32, "GGrrrrGGrrrr"),
- traci.trafficlight.Phase(2, "yyrrrryyrrrr"),
- traci.trafficlight.Phase(32, "rrGrrrrrGrrr"),
- traci.trafficlight.Phase(2, "rryrrrrryrrr"),
- traci.trafficlight.Phase(32, "rrrGGrrrrGGr"),
- traci.trafficlight.Phase(2, "rrryyrrrryyr"),
- traci.trafficlight.Phase(32, "rrrrrGrrrrrG"),
- traci.trafficlight.Phase(2, "rrrrryrrrrry")
- ])
+ max_depart_delay=0)
for run in range(1, args.runs+1):
obs = env.reset()
- agent = TrueOnlineSarsaLambda(env.observation_space, env.action_space, alpha=args.alpha, gamma=args.gamma, epsilon=args.epsilon, fourier_order=7)
+ agent = TrueOnlineSarsaLambda(env.observation_space, env.action_space, alpha=args.alpha, gamma=args.gamma, epsilon=args.epsilon, fourier_order=7, lamb=0.9)
done = False
if args.fixed:
@@ -67,7 +55,7 @@
_, _, done, _ = env.step({})
else:
while not done:
- action = agent.act(agent.get_features(obs))
+ action = agent.act(obs)
next_obs, r, done, _ = env.step(action=action)
diff --git a/experiments/sarsa_big_intersection.py b/experiments/sarsa_big_intersection.py
deleted file mode 100644
index a5d0c816..00000000
--- a/experiments/sarsa_big_intersection.py
+++ /dev/null
@@ -1,69 +0,0 @@
-import argparse
-import os
-import sys
-import pandas as pd
-from datetime import datetime
-
-if 'SUMO_HOME' in os.environ:
- tools = os.path.join(os.environ['SUMO_HOME'], 'tools')
- sys.path.append(tools)
-else:
- sys.exit("Please declare the environment variable 'SUMO_HOME'")
-
-import traci
-from sumo_rl.environment.env import SumoEnvironment
-from sumo_rl.agents.sarsa_lambda import TrueOnlineSarsaLambda
-
-
-if __name__ == '__main__':
-
- experiment_time = str(datetime.now()).split('.')[0]
- out_csv = 'outputs/big-intersection/sarsa-f7' #+ experiment_time
-
- env = SumoEnvironment(net_file='nets/big-intersection/big-intersection.net.xml',
- single_agent=True,
- route_file='nets/big-intersection/routes.rou.xml',
- out_csv_name=out_csv,
- use_gui=False,
- num_seconds=5400,
- yellow_time=4,
- min_green=5,
- max_green=60,
- max_depart_delay=300,
- time_to_load_vehicles=0,
- phases=[
- traci.trafficlight.Phase(30, "GGGGrrrrrrGGGGrrrrrr"),
- traci.trafficlight.Phase(4, "yyyyrrrrrryyyyrrrrrr"),
- traci.trafficlight.Phase(15, "rrrrGrrrrrrrrrGrrrrr"),
- traci.trafficlight.Phase(4, "rrrryrrrrrrrrryrrrrr"),
- traci.trafficlight.Phase(30, "rrrrrGGGGrrrrrrGGGGr"),
- traci.trafficlight.Phase(4, "rrrrryyyyrrrrrryyyyr"),
- traci.trafficlight.Phase(15, "rrrrrrrrrGrrrrrrrrrG"),
- traci.trafficlight.Phase(4, "rrrrrrrrryrrrrrrrrry")])
-
- fixed_tl = False
- agent = TrueOnlineSarsaLambda(env.observation_space, env.action_space, alpha=0.000000001, gamma=0.95, epsilon=0.05, lamb=0.1, fourier_order=7)
-
- for run in range(1, 4 +1):
- obs = env.reset()
- done = False
-
- if fixed_tl:
- while not done:
- _, _, done, _ = env.step(None)
-
- else:
- while not done:
- action = agent.act(agent.get_features(obs))
-
- next_obs, r, done, _ = env.step(action=action)
-
- agent.learn(state=obs, action=action, reward=r, next_state=next_obs, done=done)
-
- obs = next_obs
-
- env.save_csv(out_csv, run)
-
-
-
-
diff --git a/experiments/sarsa_double.py b/experiments/sarsa_double.py
new file mode 100644
index 00000000..0e29304b
--- /dev/null
+++ b/experiments/sarsa_double.py
@@ -0,0 +1,63 @@
+import argparse
+import os
+import sys
+import pandas as pd
+from datetime import datetime
+import fire
+
+if 'SUMO_HOME' in os.environ:
+ tools = os.path.join(os.environ['SUMO_HOME'], 'tools')
+ sys.path.append(tools)
+else:
+ sys.exit("Please declare the environment variable 'SUMO_HOME'")
+
+import traci
+from sumo_rl.environment.env import SumoEnvironment
+from linear_rl.true_online_sarsa import TrueOnlineSarsaLambda
+
+
+def run(use_gui=True, runs=1):
+ out_csv = 'outputs/double/sarsa-double'
+
+ env = SumoEnvironment(net_file='nets/double/network.net.xml',
+ single_agent=False,
+ route_file='nets/double/flow.rou.xml',
+ out_csv_name=out_csv,
+ use_gui=use_gui,
+ num_seconds=86400,
+ yellow_time=3,
+ min_green=5,
+ max_green=60,
+ max_depart_delay=300,
+ time_to_load_vehicles=0)
+
+ fixed_tl = False
+ agents = {ts_id: TrueOnlineSarsaLambda(env.observation_spaces(ts_id), env.action_spaces(ts_id), alpha=0.000000001, gamma=0.95, epsilon=0.05, lamb=0.1, fourier_order=7)
+ for ts_id in env.ts_ids}
+
+ for run in range(1, runs+1):
+ obs = env.reset()
+ done = {'__all__': False}
+
+ if fixed_tl:
+ while not done['__all__']:
+ _, _, done, _ = env.step(None)
+ else:
+ while not done['__all__']:
+ actions = {ts_id: agents[ts_id].act(obs[ts_id]) for ts_id in obs.keys()}
+
+ next_obs, r, done, _ = env.step(action=actions)
+
+ for ts_id in next_obs.keys():
+ agents[ts_id].learn(state=obs[ts_id], action=actions[ts_id], reward=r[ts_id], next_state=next_obs[ts_id], done=done[ts_id])
+ obs[ts_id] = next_obs[ts_id]
+
+ env.save_csv(out_csv, run)
+
+
+if __name__ == '__main__':
+ fire.Fire(run)
+
+
+
+
diff --git a/nets/double/flow.rou.xml b/nets/double/flow.rou.xml
new file mode 100644
index 00000000..9290c958
--- /dev/null
+++ b/nets/double/flow.rou.xml
@@ -0,0 +1,137 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/nets/double/network.net.xml b/nets/double/network.net.xml
new file mode 100644
index 00000000..2776de76
--- /dev/null
+++ b/nets/double/network.net.xml
@@ -0,0 +1,629 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+