From bffd088b227d4034d729bb404ac50aac28445bd8 Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Tue, 7 Nov 2017 15:26:40 -0700 Subject: [PATCH 01/23] added SetProcessDPIAware() to fix screen size issue for windows users --- pysc2/lib/renderer_human.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pysc2/lib/renderer_human.py b/pysc2/lib/renderer_human.py index d055536f3..0598dbb57 100644 --- a/pysc2/lib/renderer_human.py +++ b/pysc2/lib/renderer_human.py @@ -41,6 +41,10 @@ from s2clientprotocol import sc2api_pb2 as sc_pb from s2clientprotocol import spatial_pb2 as sc_spatial +import ctypes + +ctypes.windll.user32.SetProcessDPIAware() + sw = stopwatch.sw render_lock = threading.Lock() # Serialize all window/render operations. From 1a783528df249370d82942c8e8afe325d0776356 Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Thu, 9 Nov 2017 14:10:43 -0700 Subject: [PATCH 02/23] abstarting parser class --- pysc2/bin/replay_actions.py | 90 ++++++------------------- pysc2/replay_parsers/__init__.py | 13 ++++ pysc2/replay_parsers/base_parser.py | 76 +++++++++++++++++++++ pysc2/replay_parsers/random_agent.py | 34 ++++++++++ pysc2/replay_parsers/scripted_agent.py | 92 ++++++++++++++++++++++++++ 5 files changed, 234 insertions(+), 71 deletions(-) create mode 100644 pysc2/replay_parsers/__init__.py create mode 100644 pysc2/replay_parsers/base_parser.py create mode 100644 pysc2/replay_parsers/random_agent.py create mode 100644 pysc2/replay_parsers/scripted_agent.py diff --git a/pysc2/bin/replay_actions.py b/pysc2/bin/replay_actions.py index 93369a766..15484281f 100755 --- a/pysc2/bin/replay_actions.py +++ b/pysc2/bin/replay_actions.py @@ -42,10 +42,14 @@ from s2clientprotocol import common_pb2 as sc_common from s2clientprotocol import sc2api_pb2 as sc_pb +import importlib + FLAGS = flags.FLAGS flags.DEFINE_integer("parallel", 1, "How many instances to run in parallel.") flags.DEFINE_integer("step_mul", 8, "How many game steps per observation.") flags.DEFINE_string("replays", None, "Path to a directory of replays.") +flags.DEFINE_string("parser", "pysc2.replay_parsers.base_parser.BaseParser", + "Which agent to run") flags.mark_flag_as_required("replays") @@ -62,77 +66,15 @@ def sorted_dict_str(d): for k in sorted(d, key=d.get, reverse=True)) -class ReplayStats(object): - """Summary stats of the replays seen so far.""" - - def __init__(self): - self.replays = 0 - self.steps = 0 - self.camera_move = 0 - self.select_pt = 0 - self.select_rect = 0 - self.control_group = 0 - self.maps = collections.defaultdict(int) - self.races = collections.defaultdict(int) - self.unit_ids = collections.defaultdict(int) - self.valid_abilities = collections.defaultdict(int) - self.made_abilities = collections.defaultdict(int) - self.valid_actions = collections.defaultdict(int) - self.made_actions = collections.defaultdict(int) - self.crashing_replays = set() - self.invalid_replays = set() - - def merge(self, other): - """Merge another ReplayStats into this one.""" - def merge_dict(a, b): - for k, v in six.iteritems(b): - a[k] += v - - self.replays += other.replays - self.steps += other.steps - self.camera_move += other.camera_move - self.select_pt += other.select_pt - self.select_rect += other.select_rect - self.control_group += other.control_group - merge_dict(self.maps, other.maps) - merge_dict(self.races, other.races) - merge_dict(self.unit_ids, other.unit_ids) - merge_dict(self.valid_abilities, other.valid_abilities) - merge_dict(self.made_abilities, other.made_abilities) - merge_dict(self.valid_actions, other.valid_actions) - merge_dict(self.made_actions, other.made_actions) - self.crashing_replays |= other.crashing_replays - self.invalid_replays |= other.invalid_replays - - def __str__(self): - len_sorted_dict = lambda s: (len(s), sorted_dict_str(s)) - len_sorted_list = lambda s: (len(s), sorted(s)) - return "\n\n".join(( - "Replays: %s, Steps total: %s" % (self.replays, self.steps), - "Camera move: %s, Select pt: %s, Select rect: %s, Control group: %s" % ( - self.camera_move, self.select_pt, self.select_rect, - self.control_group), - "Maps: %s\n%s" % len_sorted_dict(self.maps), - "Races: %s\n%s" % len_sorted_dict(self.races), - "Unit ids: %s\n%s" % len_sorted_dict(self.unit_ids), - "Valid abilities: %s\n%s" % len_sorted_dict(self.valid_abilities), - "Made abilities: %s\n%s" % len_sorted_dict(self.made_abilities), - "Valid actions: %s\n%s" % len_sorted_dict(self.valid_actions), - "Made actions: %s\n%s" % len_sorted_dict(self.made_actions), - "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays), - "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), - )) - - class ProcessStats(object): """Stats for a worker process.""" - def __init__(self, proc_id): + def __init__(self, proc_id, parser_cls): self.proc_id = proc_id self.time = time.time() self.stage = "" self.replay = "" - self.replay_stats = ReplayStats() + self.replay_stats = parser_cls def update(self, stage): self.time = time.time() @@ -166,12 +108,13 @@ def valid_replay(info, ping): class ReplayProcessor(multiprocessing.Process): """A Process that pulls replays and processes them.""" - def __init__(self, proc_id, run_config, replay_queue, stats_queue): + def __init__(self, proc_id, run_config, replay_queue, stats_queue, parser_cls): super(ReplayProcessor, self).__init__() - self.stats = ProcessStats(proc_id) + self.stats = ProcessStats(proc_id, parser_cls) self.run_config = run_config self.replay_queue = replay_queue self.stats_queue = stats_queue + self.parser_cls = parser_cls def run(self): signal.signal(signal.SIGTERM, lambda a, b: sys.exit()) # Exit quietly. @@ -292,9 +235,9 @@ def process_replay(self, controller, replay_data, map_data, player_id): controller.step(FLAGS.step_mul) -def stats_printer(stats_queue): +def stats_printer(stats_queue, parser_cls): """A thread that consumes stats_queue and prints them every 10 seconds.""" - proc_stats = [ProcessStats(i) for i in range(FLAGS.parallel)] + proc_stats = [ProcessStats(i,parser_cls) for i in range(FLAGS.parallel)] print_time = start_time = time.time() width = 107 @@ -312,7 +255,7 @@ def stats_printer(stats_queue): except queue.Empty: pass - replay_stats = ReplayStats() + replay_stats = parser_cls for s in proc_stats: replay_stats.merge(s.replay_stats) @@ -333,11 +276,16 @@ def main(unused_argv): """Dump stats about all the actions that are in use in a set of replays.""" run_config = run_configs.get() + parser_module, parser_name = FLAGS.parser.rsplit(".", 1) + print(parser_module) + print(parser_name) + parser_cls = getattr(importlib.import_module(parser_module), parser_name) + if not gfile.Exists(FLAGS.replays): sys.exit("{} doesn't exist.".format(FLAGS.replays)) stats_queue = multiprocessing.Queue() - stats_thread = threading.Thread(target=stats_printer, args=(stats_queue,)) + stats_thread = threading.Thread(target=stats_printer, args=(stats_queue,parser_cls)) stats_thread.start() try: # For some reason buffering everything into a JoinableQueue makes the @@ -355,7 +303,7 @@ def main(unused_argv): replay_queue_thread.start() for i in range(FLAGS.parallel): - p = ReplayProcessor(i, run_config, replay_queue, stats_queue) + p = ReplayProcessor(i, run_config, replay_queue, stats_queue, parser_cls) p.daemon = True p.start() time.sleep(1) # Stagger startups, otherwise they seem to conflict somehow diff --git a/pysc2/replay_parsers/__init__.py b/pysc2/replay_parsers/__init__.py new file mode 100644 index 000000000..b448f59d9 --- /dev/null +++ b/pysc2/replay_parsers/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/pysc2/replay_parsers/base_parser.py b/pysc2/replay_parsers/base_parser.py new file mode 100644 index 000000000..168746247 --- /dev/null +++ b/pysc2/replay_parsers/base_parser.py @@ -0,0 +1,76 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A base replay parser to write custom replay data scrappers.""" + + +class BaseParser(object): + """Summary stats of the replays seen so far.""" + + def __init__(self): + self.replays = 0 + self.steps = 0 + self.camera_move = 0 + self.select_pt = 0 + self.select_rect = 0 + self.control_group = 0 + self.maps = collections.defaultdict(int) + self.races = collections.defaultdict(int) + self.unit_ids = collections.defaultdict(int) + self.valid_abilities = collections.defaultdict(int) + self.made_abilities = collections.defaultdict(int) + self.valid_actions = collections.defaultdict(int) + self.made_actions = collections.defaultdict(int) + self.crashing_replays = set() + self.invalid_replays = set() + + def merge(self, other): + """Merge another ReplayStats into this one.""" + def merge_dict(a, b): + for k, v in six.iteritems(b): + a[k] += v + + self.replays += other.replays + self.steps += other.steps + self.camera_move += other.camera_move + self.select_pt += other.select_pt + self.select_rect += other.select_rect + self.control_group += other.control_group + merge_dict(self.maps, other.maps) + merge_dict(self.races, other.races) + merge_dict(self.unit_ids, other.unit_ids) + merge_dict(self.valid_abilities, other.valid_abilities) + merge_dict(self.made_abilities, other.made_abilities) + merge_dict(self.valid_actions, other.valid_actions) + merge_dict(self.made_actions, other.made_actions) + self.crashing_replays |= other.crashing_replays + self.invalid_replays |= other.invalid_replays + + def __str__(self): + len_sorted_dict = lambda s: (len(s), sorted_dict_str(s)) + len_sorted_list = lambda s: (len(s), sorted(s)) + return "\n\n".join(( + "Replays: %s, Steps total: %s" % (self.replays, self.steps), + "Camera move: %s, Select pt: %s, Select rect: %s, Control group: %s" % ( + self.camera_move, self.select_pt, self.select_rect, + self.control_group), + "Maps: %s\n%s" % len_sorted_dict(self.maps), + "Races: %s\n%s" % len_sorted_dict(self.races), + "Unit ids: %s\n%s" % len_sorted_dict(self.unit_ids), + "Valid abilities: %s\n%s" % len_sorted_dict(self.valid_abilities), + "Made abilities: %s\n%s" % len_sorted_dict(self.made_abilities), + "Valid actions: %s\n%s" % len_sorted_dict(self.valid_actions), + "Made actions: %s\n%s" % len_sorted_dict(self.made_actions), + "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays), + "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), + )) diff --git a/pysc2/replay_parsers/random_agent.py b/pysc2/replay_parsers/random_agent.py new file mode 100644 index 000000000..b618cf916 --- /dev/null +++ b/pysc2/replay_parsers/random_agent.py @@ -0,0 +1,34 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A random agent for starcraft.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy + +from pysc2.agents import base_agent +from pysc2.lib import actions + + +class RandomAgent(base_agent.BaseAgent): + """A random agent for starcraft.""" + + def step(self, obs): + super(RandomAgent, self).step(obs) + function_id = numpy.random.choice(obs.observation["available_actions"]) + args = [[numpy.random.randint(0, size) for size in arg.sizes] + for arg in self.action_spec.functions[function_id].args] + return actions.FunctionCall(function_id, args) diff --git a/pysc2/replay_parsers/scripted_agent.py b/pysc2/replay_parsers/scripted_agent.py new file mode 100644 index 000000000..2acecb263 --- /dev/null +++ b/pysc2/replay_parsers/scripted_agent.py @@ -0,0 +1,92 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Scripted agents.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import numpy + +from pysc2.agents import base_agent +from pysc2.lib import actions +from pysc2.lib import features + +_PLAYER_RELATIVE = features.SCREEN_FEATURES.player_relative.index +_PLAYER_FRIENDLY = 1 +_PLAYER_NEUTRAL = 3 # beacon/minerals +_PLAYER_HOSTILE = 4 +_NO_OP = actions.FUNCTIONS.no_op.id +_MOVE_SCREEN = actions.FUNCTIONS.Move_screen.id +_ATTACK_SCREEN = actions.FUNCTIONS.Attack_screen.id +_SELECT_ARMY = actions.FUNCTIONS.select_army.id +_NOT_QUEUED = [0] +_SELECT_ALL = [0] + + +class MoveToBeacon(base_agent.BaseAgent): + """An agent specifically for solving the MoveToBeacon map.""" + + def step(self, obs): + super(MoveToBeacon, self).step(obs) + if _MOVE_SCREEN in obs.observation["available_actions"]: + player_relative = obs.observation["screen"][_PLAYER_RELATIVE] + neutral_y, neutral_x = (player_relative == _PLAYER_NEUTRAL).nonzero() + if not neutral_y.any(): + return actions.FunctionCall(_NO_OP, []) + target = [int(neutral_x.mean()), int(neutral_y.mean())] + return actions.FunctionCall(_MOVE_SCREEN, [_NOT_QUEUED, target]) + else: + return actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL]) + + +class CollectMineralShards(base_agent.BaseAgent): + """An agent specifically for solving the CollectMineralShards map.""" + + def step(self, obs): + super(CollectMineralShards, self).step(obs) + if _MOVE_SCREEN in obs.observation["available_actions"]: + player_relative = obs.observation["screen"][_PLAYER_RELATIVE] + neutral_y, neutral_x = (player_relative == _PLAYER_NEUTRAL).nonzero() + player_y, player_x = (player_relative == _PLAYER_FRIENDLY).nonzero() + if not neutral_y.any() or not player_y.any(): + return actions.FunctionCall(_NO_OP, []) + player = [int(player_x.mean()), int(player_y.mean())] + closest, min_dist = None, None + for p in zip(neutral_x, neutral_y): + dist = numpy.linalg.norm(numpy.array(player) - numpy.array(p)) + if not min_dist or dist < min_dist: + closest, min_dist = p, dist + return actions.FunctionCall(_MOVE_SCREEN, [_NOT_QUEUED, closest]) + else: + return actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL]) + + +class DefeatRoaches(base_agent.BaseAgent): + """An agent specifically for solving the DefeatRoaches map.""" + + def step(self, obs): + super(DefeatRoaches, self).step(obs) + if _ATTACK_SCREEN in obs.observation["available_actions"]: + player_relative = obs.observation["screen"][_PLAYER_RELATIVE] + roach_y, roach_x = (player_relative == _PLAYER_HOSTILE).nonzero() + if not roach_y.any(): + return actions.FunctionCall(_NO_OP, []) + index = numpy.argmax(roach_y) + target = [roach_x[index], roach_y[index]] + return actions.FunctionCall(_ATTACK_SCREEN, [_NOT_QUEUED, target]) + elif _SELECT_ARMY in obs.observation["available_actions"]: + return actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL]) + else: + return actions.FunctionCall(_NO_OP, []) From 3eb8869f582e45284f43f9f74a4b691711bf739c Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Thu, 9 Nov 2017 14:50:45 -0700 Subject: [PATCH 03/23] replar parser abstracted, analog to agents abstraction --- pysc2/bin/replay_actions.py | 11 +++-------- pysc2/replay_parsers/base_parser.py | 6 ++++++ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pysc2/bin/replay_actions.py b/pysc2/bin/replay_actions.py index 15484281f..c222d6fd8 100755 --- a/pysc2/bin/replay_actions.py +++ b/pysc2/bin/replay_actions.py @@ -61,11 +61,6 @@ size.assign_to(interface.feature_layer.minimap_resolution) -def sorted_dict_str(d): - return "{%s}" % ", ".join("%s: %s" % (k, d[k]) - for k in sorted(d, key=d.get, reverse=True)) - - class ProcessStats(object): """Stats for a worker process.""" @@ -74,7 +69,7 @@ def __init__(self, proc_id, parser_cls): self.time = time.time() self.stage = "" self.replay = "" - self.replay_stats = parser_cls + self.replay_stats = parser_cls() def update(self, stage): self.time = time.time() @@ -114,7 +109,7 @@ def __init__(self, proc_id, run_config, replay_queue, stats_queue, parser_cls): self.run_config = run_config self.replay_queue = replay_queue self.stats_queue = stats_queue - self.parser_cls = parser_cls + self.parser_cls = parser_cls() def run(self): signal.signal(signal.SIGTERM, lambda a, b: sys.exit()) # Exit quietly. @@ -255,7 +250,7 @@ def stats_printer(stats_queue, parser_cls): except queue.Empty: pass - replay_stats = parser_cls + replay_stats = parser_cls() for s in proc_stats: replay_stats.merge(s.replay_stats) diff --git a/pysc2/replay_parsers/base_parser.py b/pysc2/replay_parsers/base_parser.py index 168746247..95d77da25 100644 --- a/pysc2/replay_parsers/base_parser.py +++ b/pysc2/replay_parsers/base_parser.py @@ -13,6 +13,12 @@ # limitations under the License. """A base replay parser to write custom replay data scrappers.""" +import collections +import six + +def sorted_dict_str(d): + return "{%s}" % ", ".join("%s: %s" % (k, d[k]) + for k in sorted(d, key=d.get, reverse=True)) class BaseParser(object): """Summary stats of the replays seen so far.""" From 1bdf50272d8b2bb9993a2c8023a76a5dad5aa46c Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Thu, 9 Nov 2017 15:47:33 -0700 Subject: [PATCH 04/23] further abstraction of replay processing --- pysc2/bin/replay_actions.py | 76 ++++------------ pysc2/replay_parsers/base_parser.py | 134 ++++++++++++++++++---------- 2 files changed, 104 insertions(+), 106 deletions(-) diff --git a/pysc2/bin/replay_actions.py b/pysc2/bin/replay_actions.py index c222d6fd8..4a862eb94 100755 --- a/pysc2/bin/replay_actions.py +++ b/pysc2/bin/replay_actions.py @@ -69,7 +69,7 @@ def __init__(self, proc_id, parser_cls): self.time = time.time() self.stage = "" self.replay = "" - self.replay_stats = parser_cls() + self.parser = parser_cls() def update(self, stage): self.time = time.time() @@ -78,28 +78,12 @@ def update(self, stage): def __str__(self): return ("[%2d] replay: %10s, replays: %5d, steps: %7d, game loops: %7s, " "last: %12s, %3d s ago" % ( - self.proc_id, self.replay, self.replay_stats.replays, - self.replay_stats.steps, - self.replay_stats.steps * FLAGS.step_mul, self.stage, + self.proc_id, self.replay, self.parser.replays, + self.parser.steps, + self.parser.steps * FLAGS.step_mul, self.stage, time.time() - self.time)) -def valid_replay(info, ping): - """Make sure the replay isn't corrupt, and is worth looking at.""" - if (info.HasField("error") or - info.base_build != ping.base_build or # different game version - info.game_duration_loops < 1000 or - len(info.player_info) != 2): - # Probably corrupt, or just not interesting. - return False - for p in info.player_info: - if p.player_apm < 10 or p.player_mmr < 1000: - # Low APM = player just standing around. - # Low MMR = corrupt replay or player who is weak. - return False - return True - - class ReplayProcessor(multiprocessing.Process): """A Process that pulls replays and processes them.""" @@ -109,7 +93,6 @@ def __init__(self, proc_id, run_config, replay_queue, stats_queue, parser_cls): self.run_config = run_config self.replay_queue = replay_queue self.stats_queue = stats_queue - self.parser_cls = parser_cls() def run(self): signal.signal(signal.SIGTERM, lambda a, b: sys.exit()) # Exit quietly. @@ -140,12 +123,12 @@ def run(self): self._print((" Replay Info %s " % replay_name).center(60, "-")) self._print(info) self._print("-" * 60) - if valid_replay(info, ping): - self.stats.replay_stats.maps[info.map_name] += 1 + if self.stats.parser.valid_replay(info, ping): + self.stats.parser.maps[info.map_name] += 1 for player_info in info.player_info: race_name = sc_common.Race.Name( player_info.player_info.race_actual) - self.stats.replay_stats.races[race_name] += 1 + self.stats.parser.races[race_name] += 1 map_data = None if info.local_map_path: self._update_stage("open map file") @@ -157,13 +140,13 @@ def run(self): player_id) else: self._print("Replay is invalid.") - self.stats.replay_stats.invalid_replays.add(replay_name) + self.stats.parser.invalid_replays.add(replay_name) finally: self.replay_queue.task_done() self._update_stage("shutdown") except (protocol.ConnectionError, protocol.ProtocolError, remote_controller.RequestError): - self.stats.replay_stats.crashing_replays.add(replay_name) + self.stats.parser.crashing_replays.add(replay_name) except KeyboardInterrupt: return @@ -186,42 +169,15 @@ def process_replay(self, controller, replay_data, map_data, player_id): feat = features.Features(controller.game_info()) - self.stats.replay_stats.replays += 1 + self.stats.parser.replays += 1 self._update_stage("step") controller.step() while True: - self.stats.replay_stats.steps += 1 + self.stats.parser.steps += 1 self._update_stage("observe") obs = controller.observe() - for action in obs.actions: - act_fl = action.action_feature_layer - if act_fl.HasField("unit_command"): - self.stats.replay_stats.made_abilities[ - act_fl.unit_command.ability_id] += 1 - if act_fl.HasField("camera_move"): - self.stats.replay_stats.camera_move += 1 - if act_fl.HasField("unit_selection_point"): - self.stats.replay_stats.select_pt += 1 - if act_fl.HasField("unit_selection_rect"): - self.stats.replay_stats.select_rect += 1 - if action.action_ui.HasField("control_group"): - self.stats.replay_stats.control_group += 1 - - try: - func = feat.reverse_action(action).function - except ValueError: - func = -1 - self.stats.replay_stats.made_actions[func] += 1 - - for valid in obs.observation.abilities: - self.stats.replay_stats.valid_abilities[valid.ability_id] += 1 - - for u in obs.observation.raw_data.units: - self.stats.replay_stats.unit_ids[u.unit_type] += 1 - - for ability_id in feat.available_actions(obs.observation): - self.stats.replay_stats.valid_actions[ability_id] += 1 + self.stats.parser.parse_step(obs,feat) if obs.player_result: break @@ -250,12 +206,12 @@ def stats_printer(stats_queue, parser_cls): except queue.Empty: pass - replay_stats = parser_cls() + parser = parser_cls() for s in proc_stats: - replay_stats.merge(s.replay_stats) + parser.merge(s.parser) print((" Summary %0d secs " % (print_time - start_time)).center(width, "=")) - print(replay_stats) + print(parser) print(" Process stats ".center(width, "-")) print("\n".join(str(s) for s in proc_stats)) print("=" * width) @@ -272,8 +228,6 @@ def main(unused_argv): run_config = run_configs.get() parser_module, parser_name = FLAGS.parser.rsplit(".", 1) - print(parser_module) - print(parser_name) parser_cls = getattr(importlib.import_module(parser_module), parser_name) if not gfile.Exists(FLAGS.replays): diff --git a/pysc2/replay_parsers/base_parser.py b/pysc2/replay_parsers/base_parser.py index 95d77da25..a05db6d9e 100644 --- a/pysc2/replay_parsers/base_parser.py +++ b/pysc2/replay_parsers/base_parser.py @@ -21,55 +21,54 @@ def sorted_dict_str(d): for k in sorted(d, key=d.get, reverse=True)) class BaseParser(object): - """Summary stats of the replays seen so far.""" + """Summary stats of the replays seen so far.""" + def __init__(self): + self.replays = 0 + self.steps = 0 + self.camera_move = 0 + self.select_pt = 0 + self.select_rect = 0 + self.control_group = 0 + self.maps = collections.defaultdict(int) + self.races = collections.defaultdict(int) + self.unit_ids = collections.defaultdict(int) + self.valid_abilities = collections.defaultdict(int) + self.made_abilities = collections.defaultdict(int) + self.valid_actions = collections.defaultdict(int) + self.made_actions = collections.defaultdict(int) + self.crashing_replays = set() + self.invalid_replays = set() - def __init__(self): - self.replays = 0 - self.steps = 0 - self.camera_move = 0 - self.select_pt = 0 - self.select_rect = 0 - self.control_group = 0 - self.maps = collections.defaultdict(int) - self.races = collections.defaultdict(int) - self.unit_ids = collections.defaultdict(int) - self.valid_abilities = collections.defaultdict(int) - self.made_abilities = collections.defaultdict(int) - self.valid_actions = collections.defaultdict(int) - self.made_actions = collections.defaultdict(int) - self.crashing_replays = set() - self.invalid_replays = set() + def merge(self, other): + """Merge another ReplayStats into this one.""" + def merge_dict(a, b): + for k, v in six.iteritems(b): + a[k] += v - def merge(self, other): - """Merge another ReplayStats into this one.""" - def merge_dict(a, b): - for k, v in six.iteritems(b): - a[k] += v + self.replays += other.replays + self.steps += other.steps + self.camera_move += other.camera_move + self.select_pt += other.select_pt + self.select_rect += other.select_rect + self.control_group += other.control_group + merge_dict(self.maps, other.maps) + merge_dict(self.races, other.races) + merge_dict(self.unit_ids, other.unit_ids) + merge_dict(self.valid_abilities, other.valid_abilities) + merge_dict(self.made_abilities, other.made_abilities) + merge_dict(self.valid_actions, other.valid_actions) + merge_dict(self.made_actions, other.made_actions) + self.crashing_replays |= other.crashing_replays + self.invalid_replays |= other.invalid_replays - self.replays += other.replays - self.steps += other.steps - self.camera_move += other.camera_move - self.select_pt += other.select_pt - self.select_rect += other.select_rect - self.control_group += other.control_group - merge_dict(self.maps, other.maps) - merge_dict(self.races, other.races) - merge_dict(self.unit_ids, other.unit_ids) - merge_dict(self.valid_abilities, other.valid_abilities) - merge_dict(self.made_abilities, other.made_abilities) - merge_dict(self.valid_actions, other.valid_actions) - merge_dict(self.made_actions, other.made_actions) - self.crashing_replays |= other.crashing_replays - self.invalid_replays |= other.invalid_replays - - def __str__(self): - len_sorted_dict = lambda s: (len(s), sorted_dict_str(s)) - len_sorted_list = lambda s: (len(s), sorted(s)) - return "\n\n".join(( + def __str__(self): + len_sorted_dict = lambda s: (len(s), sorted_dict_str(s)) + len_sorted_list = lambda s: (len(s), sorted(s)) + return "\n\n".join(( "Replays: %s, Steps total: %s" % (self.replays, self.steps), "Camera move: %s, Select pt: %s, Select rect: %s, Control group: %s" % ( - self.camera_move, self.select_pt, self.select_rect, - self.control_group), + self.camera_move, self.select_pt, self.select_rect, + self.control_group), "Maps: %s\n%s" % len_sorted_dict(self.maps), "Races: %s\n%s" % len_sorted_dict(self.races), "Unit ids: %s\n%s" % len_sorted_dict(self.unit_ids), @@ -79,4 +78,49 @@ def __str__(self): "Made actions: %s\n%s" % len_sorted_dict(self.made_actions), "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays), "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), - )) + )) + + def valid_replay(self,info, ping): + """Make sure the replay isn't corrupt, and is worth looking at.""" + if (info.HasField("error") or + info.base_build != ping.base_build or # different game version + info.game_duration_loops < 1000 or + len(info.player_info) != 2): + # Probably corrupt, or just not interesting. + return False + for p in info.player_info: + if p.player_apm < 10 or p.player_mmr < 1000: + # Low APM = player just standing around. + # Low MMR = corrupt replay or player who is weak. + return False + return True + + def parse_step(self,obs,feat): + for action in obs.actions: + act_fl = action.action_feature_layer + if act_fl.HasField("unit_command"): + self.made_abilities[ + act_fl.unit_command.ability_id] += 1 + if act_fl.HasField("camera_move"): + self.camera_move += 1 + if act_fl.HasField("unit_selection_point"): + self.select_pt += 1 + if act_fl.HasField("unit_selection_rect"): + self.select_rect += 1 + if action.action_ui.HasField("control_group"): + self.control_group += 1 + + try: + func = feat.reverse_action(action).function + except ValueError: + func = -1 + self.made_actions[func] += 1 + + for valid in obs.observation.abilities: + self.valid_abilities[valid.ability_id] += 1 + + for u in obs.observation.raw_data.units: + self.unit_ids[u.unit_type] += 1 + + for ability_id in feat.available_actions(obs.observation): + self.valid_actions[ability_id] += 1 From d31a9e9059a54df7cf23889d4b6325ca0e1cdc53 Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Thu, 9 Nov 2017 17:25:30 -0700 Subject: [PATCH 05/23] action stats scrapper fully abstracted into seperate class using base parser class --- pysc2/bin/replay_actions.py | 2 + pysc2/replay_parsers/action_parser.py | 119 ++++++++++++++++++++++++++ pysc2/replay_parsers/base_parser.py | 90 +++---------------- pysc2/replay_parsers/random_agent.py | 34 -------- 4 files changed, 134 insertions(+), 111 deletions(-) create mode 100644 pysc2/replay_parsers/action_parser.py delete mode 100644 pysc2/replay_parsers/random_agent.py diff --git a/pysc2/bin/replay_actions.py b/pysc2/bin/replay_actions.py index 4a862eb94..7323156b4 100755 --- a/pysc2/bin/replay_actions.py +++ b/pysc2/bin/replay_actions.py @@ -43,6 +43,7 @@ from s2clientprotocol import sc2api_pb2 as sc_pb import importlib +import sys FLAGS = flags.FLAGS flags.DEFINE_integer("parallel", 1, "How many instances to run in parallel.") @@ -51,6 +52,7 @@ flags.DEFINE_string("parser", "pysc2.replay_parsers.base_parser.BaseParser", "Which agent to run") flags.mark_flag_as_required("replays") +FLAGS(sys.argv) size = point.Point(16, 16) diff --git a/pysc2/replay_parsers/action_parser.py b/pysc2/replay_parsers/action_parser.py new file mode 100644 index 000000000..b7c7a5f11 --- /dev/null +++ b/pysc2/replay_parsers/action_parser.py @@ -0,0 +1,119 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Action statistics parser for replays.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import six + +from pysc2.replay_parsers import base_parser + +class ActionParser(base_parser.BaseParser): + """Action statistics parser for replays.""" + + def __init__(self): + super(ActionParser, self).__init__() + self.camera_move = 0 + self.select_pt = 0 + self.select_rect = 0 + self.control_group = 0 + self.unit_ids = collections.defaultdict(int) + self.valid_abilities = collections.defaultdict(int) + self.made_abilities = collections.defaultdict(int) + self.valid_actions = collections.defaultdict(int) + self.made_actions = collections.defaultdict(int) + + def merge(self, other): + """Merge another ReplayStats into this one.""" + def merge_dict(a, b): + for k, v in six.iteritems(b): + a[k] += v + super(ActionParser,self).merge(other) + self.camera_move += other.camera_move + self.select_pt += other.select_pt + self.select_rect += other.select_rect + self.control_group += other.control_group + merge_dict(self.unit_ids, other.unit_ids) + merge_dict(self.valid_abilities, other.valid_abilities) + merge_dict(self.made_abilities, other.made_abilities) + merge_dict(self.valid_actions, other.valid_actions) + merge_dict(self.made_actions, other.made_actions) + + def valid_replay(self,info, ping): + return True + """Make sure the replay isn't corrupt, and is worth looking at.""" + if (info.HasField("error") or + info.base_build != ping.base_build or # different game version + info.game_duration_loops < 1000 or + len(info.player_info) != 2): + # Probably corrupt, or just not interesting. + return False + for p in info.player_info: + if p.player_apm < 10 or p.player_mmr < 1000: + # Low APM = player just standing around. + # Low MMR = corrupt replay or player who is weak. + return False + return True + + def __str__(self): + len_sorted_dict = lambda s: (len(s), self.sorted_dict_str(s)) + len_sorted_list = lambda s: (len(s), sorted(s)) + return "\n\n".join(( + "Replays: %s, Steps total: %s" % (self.replays, self.steps), + "Camera move: %s, Select pt: %s, Select rect: %s, Control group: %s" % ( + self.camera_move, self.select_pt, self.select_rect, + self.control_group), + "Maps: %s\n%s" % len_sorted_dict(self.maps), + "Races: %s\n%s" % len_sorted_dict(self.races), + "Unit ids: %s\n%s" % len_sorted_dict(self.unit_ids), + "Valid abilities: %s\n%s" % len_sorted_dict(self.valid_abilities), + "Made abilities: %s\n%s" % len_sorted_dict(self.made_abilities), + "Valid actions: %s\n%s" % len_sorted_dict(self.valid_actions), + "Made actions: %s\n%s" % len_sorted_dict(self.made_actions), + "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays), + "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), + )) + + def parse_step(self,obs,feat): + for action in obs.actions: + act_fl = action.action_feature_layer + if act_fl.HasField("unit_command"): + self.made_abilities[ + act_fl.unit_command.ability_id] += 1 + if act_fl.HasField("camera_move"): + self.camera_move += 1 + if act_fl.HasField("unit_selection_point"): + self.select_pt += 1 + if act_fl.HasField("unit_selection_rect"): + self.select_rect += 1 + if action.action_ui.HasField("control_group"): + self.control_group += 1 + + try: + func = feat.reverse_action(action).function + except ValueError: + func = -1 + self.made_actions[func] += 1 + + for valid in obs.observation.abilities: + self.valid_abilities[valid.ability_id] += 1 + + for u in obs.observation.raw_data.units: + self.unit_ids[u.unit_type] += 1 + + for ability_id in feat.available_actions(obs.observation): + self.valid_actions[ability_id] += 1 \ No newline at end of file diff --git a/pysc2/replay_parsers/base_parser.py b/pysc2/replay_parsers/base_parser.py index a05db6d9e..b42c18a49 100644 --- a/pysc2/replay_parsers/base_parser.py +++ b/pysc2/replay_parsers/base_parser.py @@ -16,111 +16,47 @@ import collections import six -def sorted_dict_str(d): - return "{%s}" % ", ".join("%s: %s" % (k, d[k]) - for k in sorted(d, key=d.get, reverse=True)) - class BaseParser(object): """Summary stats of the replays seen so far.""" def __init__(self): self.replays = 0 self.steps = 0 - self.camera_move = 0 - self.select_pt = 0 - self.select_rect = 0 - self.control_group = 0 self.maps = collections.defaultdict(int) self.races = collections.defaultdict(int) - self.unit_ids = collections.defaultdict(int) - self.valid_abilities = collections.defaultdict(int) - self.made_abilities = collections.defaultdict(int) - self.valid_actions = collections.defaultdict(int) - self.made_actions = collections.defaultdict(int) self.crashing_replays = set() self.invalid_replays = set() def merge(self, other): """Merge another ReplayStats into this one.""" + def merge_dict(a, b): for k, v in six.iteritems(b): a[k] += v - - self.replays += other.replays - self.steps += other.steps - self.camera_move += other.camera_move - self.select_pt += other.select_pt - self.select_rect += other.select_rect - self.control_group += other.control_group - merge_dict(self.maps, other.maps) - merge_dict(self.races, other.races) - merge_dict(self.unit_ids, other.unit_ids) - merge_dict(self.valid_abilities, other.valid_abilities) - merge_dict(self.made_abilities, other.made_abilities) - merge_dict(self.valid_actions, other.valid_actions) - merge_dict(self.made_actions, other.made_actions) - self.crashing_replays |= other.crashing_replays - self.invalid_replays |= other.invalid_replays + self.replays += other.replays + self.steps += other.steps + merge_dict(self.maps, other.maps) + merge_dict(self.races, other.races) + self.crashing_replays |= other.crashing_replays + self.invalid_replays |= other.invalid_replays def __str__(self): - len_sorted_dict = lambda s: (len(s), sorted_dict_str(s)) + len_sorted_dict = lambda s: (len(s), self.sorted_dict_str(s)) len_sorted_list = lambda s: (len(s), sorted(s)) return "\n\n".join(( "Replays: %s, Steps total: %s" % (self.replays, self.steps), - "Camera move: %s, Select pt: %s, Select rect: %s, Control group: %s" % ( - self.camera_move, self.select_pt, self.select_rect, - self.control_group), "Maps: %s\n%s" % len_sorted_dict(self.maps), "Races: %s\n%s" % len_sorted_dict(self.races), - "Unit ids: %s\n%s" % len_sorted_dict(self.unit_ids), - "Valid abilities: %s\n%s" % len_sorted_dict(self.valid_abilities), - "Made abilities: %s\n%s" % len_sorted_dict(self.made_abilities), - "Valid actions: %s\n%s" % len_sorted_dict(self.valid_actions), - "Made actions: %s\n%s" % len_sorted_dict(self.made_actions), "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays), "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), )) def valid_replay(self,info, ping): - """Make sure the replay isn't corrupt, and is worth looking at.""" - if (info.HasField("error") or - info.base_build != ping.base_build or # different game version - info.game_duration_loops < 1000 or - len(info.player_info) != 2): - # Probably corrupt, or just not interesting. - return False - for p in info.player_info: - if p.player_apm < 10 or p.player_mmr < 1000: - # Low APM = player just standing around. - # Low MMR = corrupt replay or player who is weak. - return False + #All replays are valid in the base parser return True def parse_step(self,obs,feat): - for action in obs.actions: - act_fl = action.action_feature_layer - if act_fl.HasField("unit_command"): - self.made_abilities[ - act_fl.unit_command.ability_id] += 1 - if act_fl.HasField("camera_move"): - self.camera_move += 1 - if act_fl.HasField("unit_selection_point"): - self.select_pt += 1 - if act_fl.HasField("unit_selection_rect"): - self.select_rect += 1 - if action.action_ui.HasField("control_group"): - self.control_group += 1 - - try: - func = feat.reverse_action(action).function - except ValueError: - func = -1 - self.made_actions[func] += 1 - - for valid in obs.observation.abilities: - self.valid_abilities[valid.ability_id] += 1 - - for u in obs.observation.raw_data.units: - self.unit_ids[u.unit_type] += 1 + pass - for ability_id in feat.available_actions(obs.observation): - self.valid_actions[ability_id] += 1 + def sorted_dict_str(self, d): + return "{%s}" % ", ".join("%s: %s" % (k, d[k]) + for k in sorted(d, key=d.get, reverse=True)) diff --git a/pysc2/replay_parsers/random_agent.py b/pysc2/replay_parsers/random_agent.py deleted file mode 100644 index b618cf916..000000000 --- a/pysc2/replay_parsers/random_agent.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2017 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS-IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""A random agent for starcraft.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy - -from pysc2.agents import base_agent -from pysc2.lib import actions - - -class RandomAgent(base_agent.BaseAgent): - """A random agent for starcraft.""" - - def step(self, obs): - super(RandomAgent, self).step(obs) - function_id = numpy.random.choice(obs.observation["available_actions"]) - args = [[numpy.random.randint(0, size) for size in arg.sizes] - for arg in self.action_spec.functions[function_id].args] - return actions.FunctionCall(function_id, args) From ad28d782fb09004fc0e8b6016347a70fe6920b76 Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Thu, 9 Nov 2017 17:41:35 -0700 Subject: [PATCH 06/23] updated args to allow user inputter feature resolutions, same as play module --- pysc2/bin/replay_actions.py | 24 +++-- pysc2/replay_parsers/base_parser.py | 3 + pysc2/replay_parsers/scripted_agent.py | 92 ------------------- pysc2/replay_parsers/state_parser.py | 119 +++++++++++++++++++++++++ 4 files changed, 138 insertions(+), 100 deletions(-) delete mode 100644 pysc2/replay_parsers/scripted_agent.py create mode 100644 pysc2/replay_parsers/state_parser.py diff --git a/pysc2/bin/replay_actions.py b/pysc2/bin/replay_actions.py index 7323156b4..93cb08d7d 100755 --- a/pysc2/bin/replay_actions.py +++ b/pysc2/bin/replay_actions.py @@ -51,17 +51,23 @@ flags.DEFINE_string("replays", None, "Path to a directory of replays.") flags.DEFINE_string("parser", "pysc2.replay_parsers.base_parser.BaseParser", "Which agent to run") +flags.DEFINE_string("data_dir", "/", + "Path to directory to save replay data from replay parser") +flags.DEFINE_integer("screen_resolution", 16, + "Resolution for screen feature layers.") +flags.DEFINE_integer("minimap_resolution", 16, + "Resolution for minimap feature layers.") flags.mark_flag_as_required("replays") FLAGS(sys.argv) - -size = point.Point(16, 16) -interface = sc_pb.InterfaceOptions( - raw=True, score=False, - feature_layer=sc_pb.SpatialCameraSetup(width=24)) -size.assign_to(interface.feature_layer.resolution) -size.assign_to(interface.feature_layer.minimap_resolution) - +interface = sc_pb.InterfaceOptions() +interface.raw = True +interface.score = False +interface.feature_layer.width = 24 +interface.feature_layer.resolution.x = FLAGS.screen_resolution +interface.feature_layer.resolution.y = FLAGS.screen_resolution +interface.feature_layer.minimap_resolution.x = FLAGS.minimap_resolution +interface.feature_layer.minimap_resolution.y = FLAGS.minimap_resolution class ProcessStats(object): """Stats for a worker process.""" @@ -182,6 +188,8 @@ def process_replay(self, controller, replay_data, map_data, player_id): self.stats.parser.parse_step(obs,feat) if obs.player_result: + #save scraped replay data to file at end of replay + self.stats.parser.save_data(FLAGS.data_dir) break self._update_stage("step") diff --git a/pysc2/replay_parsers/base_parser.py b/pysc2/replay_parsers/base_parser.py index b42c18a49..1695e72ca 100644 --- a/pysc2/replay_parsers/base_parser.py +++ b/pysc2/replay_parsers/base_parser.py @@ -57,6 +57,9 @@ def valid_replay(self,info, ping): def parse_step(self,obs,feat): pass + def save_data(self,data_dir): + pass + def sorted_dict_str(self, d): return "{%s}" % ", ".join("%s: %s" % (k, d[k]) for k in sorted(d, key=d.get, reverse=True)) diff --git a/pysc2/replay_parsers/scripted_agent.py b/pysc2/replay_parsers/scripted_agent.py deleted file mode 100644 index 2acecb263..000000000 --- a/pysc2/replay_parsers/scripted_agent.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2017 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS-IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Scripted agents.""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import numpy - -from pysc2.agents import base_agent -from pysc2.lib import actions -from pysc2.lib import features - -_PLAYER_RELATIVE = features.SCREEN_FEATURES.player_relative.index -_PLAYER_FRIENDLY = 1 -_PLAYER_NEUTRAL = 3 # beacon/minerals -_PLAYER_HOSTILE = 4 -_NO_OP = actions.FUNCTIONS.no_op.id -_MOVE_SCREEN = actions.FUNCTIONS.Move_screen.id -_ATTACK_SCREEN = actions.FUNCTIONS.Attack_screen.id -_SELECT_ARMY = actions.FUNCTIONS.select_army.id -_NOT_QUEUED = [0] -_SELECT_ALL = [0] - - -class MoveToBeacon(base_agent.BaseAgent): - """An agent specifically for solving the MoveToBeacon map.""" - - def step(self, obs): - super(MoveToBeacon, self).step(obs) - if _MOVE_SCREEN in obs.observation["available_actions"]: - player_relative = obs.observation["screen"][_PLAYER_RELATIVE] - neutral_y, neutral_x = (player_relative == _PLAYER_NEUTRAL).nonzero() - if not neutral_y.any(): - return actions.FunctionCall(_NO_OP, []) - target = [int(neutral_x.mean()), int(neutral_y.mean())] - return actions.FunctionCall(_MOVE_SCREEN, [_NOT_QUEUED, target]) - else: - return actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL]) - - -class CollectMineralShards(base_agent.BaseAgent): - """An agent specifically for solving the CollectMineralShards map.""" - - def step(self, obs): - super(CollectMineralShards, self).step(obs) - if _MOVE_SCREEN in obs.observation["available_actions"]: - player_relative = obs.observation["screen"][_PLAYER_RELATIVE] - neutral_y, neutral_x = (player_relative == _PLAYER_NEUTRAL).nonzero() - player_y, player_x = (player_relative == _PLAYER_FRIENDLY).nonzero() - if not neutral_y.any() or not player_y.any(): - return actions.FunctionCall(_NO_OP, []) - player = [int(player_x.mean()), int(player_y.mean())] - closest, min_dist = None, None - for p in zip(neutral_x, neutral_y): - dist = numpy.linalg.norm(numpy.array(player) - numpy.array(p)) - if not min_dist or dist < min_dist: - closest, min_dist = p, dist - return actions.FunctionCall(_MOVE_SCREEN, [_NOT_QUEUED, closest]) - else: - return actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL]) - - -class DefeatRoaches(base_agent.BaseAgent): - """An agent specifically for solving the DefeatRoaches map.""" - - def step(self, obs): - super(DefeatRoaches, self).step(obs) - if _ATTACK_SCREEN in obs.observation["available_actions"]: - player_relative = obs.observation["screen"][_PLAYER_RELATIVE] - roach_y, roach_x = (player_relative == _PLAYER_HOSTILE).nonzero() - if not roach_y.any(): - return actions.FunctionCall(_NO_OP, []) - index = numpy.argmax(roach_y) - target = [roach_x[index], roach_y[index]] - return actions.FunctionCall(_ATTACK_SCREEN, [_NOT_QUEUED, target]) - elif _SELECT_ARMY in obs.observation["available_actions"]: - return actions.FunctionCall(_SELECT_ARMY, [_SELECT_ALL]) - else: - return actions.FunctionCall(_NO_OP, []) diff --git a/pysc2/replay_parsers/state_parser.py b/pysc2/replay_parsers/state_parser.py new file mode 100644 index 000000000..b7c7a5f11 --- /dev/null +++ b/pysc2/replay_parsers/state_parser.py @@ -0,0 +1,119 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Action statistics parser for replays.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import six + +from pysc2.replay_parsers import base_parser + +class ActionParser(base_parser.BaseParser): + """Action statistics parser for replays.""" + + def __init__(self): + super(ActionParser, self).__init__() + self.camera_move = 0 + self.select_pt = 0 + self.select_rect = 0 + self.control_group = 0 + self.unit_ids = collections.defaultdict(int) + self.valid_abilities = collections.defaultdict(int) + self.made_abilities = collections.defaultdict(int) + self.valid_actions = collections.defaultdict(int) + self.made_actions = collections.defaultdict(int) + + def merge(self, other): + """Merge another ReplayStats into this one.""" + def merge_dict(a, b): + for k, v in six.iteritems(b): + a[k] += v + super(ActionParser,self).merge(other) + self.camera_move += other.camera_move + self.select_pt += other.select_pt + self.select_rect += other.select_rect + self.control_group += other.control_group + merge_dict(self.unit_ids, other.unit_ids) + merge_dict(self.valid_abilities, other.valid_abilities) + merge_dict(self.made_abilities, other.made_abilities) + merge_dict(self.valid_actions, other.valid_actions) + merge_dict(self.made_actions, other.made_actions) + + def valid_replay(self,info, ping): + return True + """Make sure the replay isn't corrupt, and is worth looking at.""" + if (info.HasField("error") or + info.base_build != ping.base_build or # different game version + info.game_duration_loops < 1000 or + len(info.player_info) != 2): + # Probably corrupt, or just not interesting. + return False + for p in info.player_info: + if p.player_apm < 10 or p.player_mmr < 1000: + # Low APM = player just standing around. + # Low MMR = corrupt replay or player who is weak. + return False + return True + + def __str__(self): + len_sorted_dict = lambda s: (len(s), self.sorted_dict_str(s)) + len_sorted_list = lambda s: (len(s), sorted(s)) + return "\n\n".join(( + "Replays: %s, Steps total: %s" % (self.replays, self.steps), + "Camera move: %s, Select pt: %s, Select rect: %s, Control group: %s" % ( + self.camera_move, self.select_pt, self.select_rect, + self.control_group), + "Maps: %s\n%s" % len_sorted_dict(self.maps), + "Races: %s\n%s" % len_sorted_dict(self.races), + "Unit ids: %s\n%s" % len_sorted_dict(self.unit_ids), + "Valid abilities: %s\n%s" % len_sorted_dict(self.valid_abilities), + "Made abilities: %s\n%s" % len_sorted_dict(self.made_abilities), + "Valid actions: %s\n%s" % len_sorted_dict(self.valid_actions), + "Made actions: %s\n%s" % len_sorted_dict(self.made_actions), + "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays), + "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), + )) + + def parse_step(self,obs,feat): + for action in obs.actions: + act_fl = action.action_feature_layer + if act_fl.HasField("unit_command"): + self.made_abilities[ + act_fl.unit_command.ability_id] += 1 + if act_fl.HasField("camera_move"): + self.camera_move += 1 + if act_fl.HasField("unit_selection_point"): + self.select_pt += 1 + if act_fl.HasField("unit_selection_rect"): + self.select_rect += 1 + if action.action_ui.HasField("control_group"): + self.control_group += 1 + + try: + func = feat.reverse_action(action).function + except ValueError: + func = -1 + self.made_actions[func] += 1 + + for valid in obs.observation.abilities: + self.valid_abilities[valid.ability_id] += 1 + + for u in obs.observation.raw_data.units: + self.unit_ids[u.unit_type] += 1 + + for ability_id in feat.available_actions(obs.observation): + self.valid_actions[ability_id] += 1 \ No newline at end of file From 09e32744450c0b25fa95f7ab6490aef44c16e961 Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Thu, 9 Nov 2017 19:09:44 -0700 Subject: [PATCH 07/23] added state scrapper example and ability to save to file --- pysc2/bin/replay_actions.py | 28 +++-- pysc2/replay_parsers/action_parser.py | 3 +- pysc2/replay_parsers/base_parser.py | 9 +- pysc2/replay_parsers/state_parser.py | 159 +++++++++++++++----------- 4 files changed, 114 insertions(+), 85 deletions(-) diff --git a/pysc2/bin/replay_actions.py b/pysc2/bin/replay_actions.py index 93cb08d7d..1e6c938c4 100755 --- a/pysc2/bin/replay_actions.py +++ b/pysc2/bin/replay_actions.py @@ -43,6 +43,7 @@ from s2clientprotocol import sc2api_pb2 as sc_pb import importlib +import json import sys FLAGS = flags.FLAGS @@ -51,7 +52,7 @@ flags.DEFINE_string("replays", None, "Path to a directory of replays.") flags.DEFINE_string("parser", "pysc2.replay_parsers.base_parser.BaseParser", "Which agent to run") -flags.DEFINE_string("data_dir", "/", +flags.DEFINE_string("data_dir", "C:/", "Path to directory to save replay data from replay parser") flags.DEFINE_integer("screen_resolution", 16, "Resolution for screen feature layers.") @@ -121,7 +122,7 @@ def run(self): self._print("Empty queue, returning") return try: - replay_name = os.path.basename(replay_path)[:10] + replay_name = os.path.basename(replay_path) self.stats.replay = replay_name self._print("Got replay: %s" % replay_path) self._update_stage("open replay file") @@ -145,7 +146,7 @@ def run(self): self._print("Starting %s from player %s's perspective" % ( replay_name, player_id)) self.process_replay(controller, replay_data, map_data, - player_id) + player_id,info,replay_name) else: self._print("Replay is invalid.") self.stats.parser.invalid_replays.add(replay_name) @@ -166,7 +167,7 @@ def _update_stage(self, stage): self.stats.update(stage) self.stats_queue.put(self.stats) - def process_replay(self, controller, replay_data, map_data, player_id): + def process_replay(self, controller, replay_data, map_data, player_id,info,replay_name): """Process a single replay, updating the stats.""" self._update_stage("start_replay") controller.start_replay(sc_pb.RequestStartReplay( @@ -180,16 +181,25 @@ def process_replay(self, controller, replay_data, map_data, player_id): self.stats.parser.replays += 1 self._update_stage("step") controller.step() + data = [] while True: self.stats.parser.steps += 1 self._update_stage("observe") obs = controller.observe() - self.stats.parser.parse_step(obs,feat) - - if obs.player_result: - #save scraped replay data to file at end of replay - self.stats.parser.save_data(FLAGS.data_dir) + #if parser.parse_step returns, whatever is returned is appended + #to a data list, and this data list is saved to a json file + #in the data_dir directory with filename = replay_name_player_id.json + parsed_data = self.stats.parser.parse_step(obs,feat,info) + if parsed_data: + data.append(parsed_data) + + if obs.player_result: + #save scraped replay data to file at end of replay if parser returns + if data: + data_file = FLAGS.data_dir + replay_name + "_" + str(player_id) + '.json' + with open(data_file,'w') as outfile: + json.dump(data,outfile) break self._update_stage("step") diff --git a/pysc2/replay_parsers/action_parser.py b/pysc2/replay_parsers/action_parser.py index b7c7a5f11..47c68c8a5 100644 --- a/pysc2/replay_parsers/action_parser.py +++ b/pysc2/replay_parsers/action_parser.py @@ -54,7 +54,6 @@ def merge_dict(a, b): merge_dict(self.made_actions, other.made_actions) def valid_replay(self,info, ping): - return True """Make sure the replay isn't corrupt, and is worth looking at.""" if (info.HasField("error") or info.base_build != ping.base_build or # different game version @@ -88,7 +87,7 @@ def __str__(self): "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), )) - def parse_step(self,obs,feat): + def parse_step(self,obs,feat,info): for action in obs.actions: act_fl = action.action_feature_layer if act_fl.HasField("unit_command"): diff --git a/pysc2/replay_parsers/base_parser.py b/pysc2/replay_parsers/base_parser.py index 1695e72ca..20e7676c4 100644 --- a/pysc2/replay_parsers/base_parser.py +++ b/pysc2/replay_parsers/base_parser.py @@ -17,7 +17,7 @@ import six class BaseParser(object): - """Summary stats of the replays seen so far.""" + """A base replay parser to write custom replay data scrappers.""" def __init__(self): self.replays = 0 self.steps = 0 @@ -54,10 +54,9 @@ def valid_replay(self,info, ping): #All replays are valid in the base parser return True - def parse_step(self,obs,feat): - pass - - def save_data(self,data_dir): + def parse_step(self,obs,feat,info): + #base parser doesn't directly parse any data, + #parse_step is a required function for parsers pass def sorted_dict_str(self, d): diff --git a/pysc2/replay_parsers/state_parser.py b/pysc2/replay_parsers/state_parser.py index b7c7a5f11..1db287e48 100644 --- a/pysc2/replay_parsers/state_parser.py +++ b/pysc2/replay_parsers/state_parser.py @@ -19,40 +19,68 @@ import collections import six +import numpy as np from pysc2.replay_parsers import base_parser -class ActionParser(base_parser.BaseParser): - """Action statistics parser for replays.""" +from s2clientprotocol import sc2api_pb2 as sc_pb +from s2clientprotocol import common_pb2 as sc_common + +def calc_armies(screen): + friendly_army = [] + enemy_army = [] + unit_list = np.unique(screen[6]) + for unit in unit_list: + friendly_pixels = (screen[5] == 1) & (screen[6] == unit) + friendly_unit_count = sum(screen[11,friendly_pixels]) + #only append if count > 0 + if friendly_unit_count: + friendly_army.append([int(unit),friendly_unit_count]) + enemy_pixels = (screen[5] == 4) & (screen[6] == unit) + enemy_unit_count = sum(screen[11,enemy_pixels]) + if enemy_unit_count: + enemy_army.append([int(unit), enemy_unit_count]) + return friendly_army, enemy_army - def __init__(self): - super(ActionParser, self).__init__() - self.camera_move = 0 - self.select_pt = 0 - self.select_rect = 0 - self.control_group = 0 - self.unit_ids = collections.defaultdict(int) - self.valid_abilities = collections.defaultdict(int) - self.made_abilities = collections.defaultdict(int) - self.valid_actions = collections.defaultdict(int) - self.made_actions = collections.defaultdict(int) +def update_minimap(minimap,screen): + #Update minimap data with screen details + #Identify which minimap squares are on screen + visible = minimap[1] == 1 + #TODO: need to devide screen into visible minimap, for now + #devide each quantity by number of visible minimap squares + total_visible = sum(visible.ravel()) + #power + minimap[4,visible] = (sum(screen[3].ravel())/ + (len(screen[3].ravel())*total_visible)) + #friendy army + friendly_units = screen[5] == 1 + #unit density + minimap[5,visible] = sum(screen[11,friendly_units])/total_visible + #Most common unit + if friendly_units.any() == True: + minimap[6,visible] = np.bincount(screen[6,friendly_units]).argmax() + else: + minimap[6,visible] = 0 + #Total HP + Shields + minimap[7,visible] = ((sum(screen[8,friendly_units]) + + sum(screen[10,friendly_units]))/total_visible) + #enemy army + enemy_units = screen[5] == 4 + #unit density + minimap[8,visible] = sum(screen[11,enemy_units])/total_visible + #main unit + if enemy_units.any() == True: + minimap[9,visible] = np.bincount(screen[6,enemy_units]).argmax() + else: + minimap[9,visible] = 0 + #Total HP + shields + minimap[10,visible] = ((sum(screen[8,enemy_units]) + + sum(screen[10,friendly_units]))/total_visible) - def merge(self, other): - """Merge another ReplayStats into this one.""" - def merge_dict(a, b): - for k, v in six.iteritems(b): - a[k] += v - super(ActionParser,self).merge(other) - self.camera_move += other.camera_move - self.select_pt += other.select_pt - self.select_rect += other.select_rect - self.control_group += other.control_group - merge_dict(self.unit_ids, other.unit_ids) - merge_dict(self.valid_abilities, other.valid_abilities) - merge_dict(self.made_abilities, other.made_abilities) - merge_dict(self.valid_actions, other.valid_actions) - merge_dict(self.made_actions, other.made_actions) + return minimap +class StateParser(base_parser.BaseParser): + """Action statistics parser for replays.""" def valid_replay(self,info, ping): return True """Make sure the replay isn't corrupt, and is worth looking at.""" @@ -69,51 +97,44 @@ def valid_replay(self,info, ping): return False return True - def __str__(self): - len_sorted_dict = lambda s: (len(s), self.sorted_dict_str(s)) - len_sorted_list = lambda s: (len(s), sorted(s)) - return "\n\n".join(( - "Replays: %s, Steps total: %s" % (self.replays, self.steps), - "Camera move: %s, Select pt: %s, Select rect: %s, Control group: %s" % ( - self.camera_move, self.select_pt, self.select_rect, - self.control_group), - "Maps: %s\n%s" % len_sorted_dict(self.maps), - "Races: %s\n%s" % len_sorted_dict(self.races), - "Unit ids: %s\n%s" % len_sorted_dict(self.unit_ids), - "Valid abilities: %s\n%s" % len_sorted_dict(self.valid_abilities), - "Made abilities: %s\n%s" % len_sorted_dict(self.made_abilities), - "Valid actions: %s\n%s" % len_sorted_dict(self.valid_actions), - "Made actions: %s\n%s" % len_sorted_dict(self.made_actions), - "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays), - "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), - )) - - def parse_step(self,obs,feat): + def parse_step(self,obs,feat,info): + actions = [] for action in obs.actions: - act_fl = action.action_feature_layer - if act_fl.HasField("unit_command"): - self.made_abilities[ - act_fl.unit_command.ability_id] += 1 - if act_fl.HasField("camera_move"): - self.camera_move += 1 - if act_fl.HasField("unit_selection_point"): - self.select_pt += 1 - if act_fl.HasField("unit_selection_rect"): - self.select_rect += 1 - if action.action_ui.HasField("control_group"): - self.control_group += 1 - try: - func = feat.reverse_action(action).function + full_act = feat.reverse_action(action) + func = full_act.function + args = full_act.arguments except ValueError: func = -1 - self.made_actions[func] += 1 + args = [] + + actions.append([func,args]) - for valid in obs.observation.abilities: - self.valid_abilities[valid.ability_id] += 1 + all_features = feat.transform_obs(obs.observation) + #remove elevation, viz and selected data from minimap + minimap_data = all_features['minimap'][2:6,:,:] + screen = all_features['screen'] - for u in obs.observation.raw_data.units: - self.unit_ids[u.unit_type] += 1 + mini_shape = minimap_data.shape + minimap = np.zeros(shape=(11,mini_shape[1],mini_shape[2]),dtype=np.int) + minimap[0:4,:,:] = minimap_data + extended_minimap = update_minimap(minimap,screen).tolist() + friendly_army, enemy_army = calc_armies(screen) - for ability_id in feat.available_actions(obs.observation): - self.valid_actions[ability_id] += 1 \ No newline at end of file + if info.player_info[0].player_result.result == 'Victory': + winner = 1 + else: + winner = 2 + for player_id in [1, 2]: + race = sc_common.Race.Name(info.player_info[player_id-1].player_info.race_actual) + if player_id == 1: + enemy = 2 + else: + enemy = 1 + enemy_race = sc_common.Race.Name(info.player_info[enemy-1].player_info.race_actual) + + full_state = [info.map_name, extended_minimap, + friendly_army,enemy_army,all_features['player'].tolist(), + all_features['available_actions'].tolist(),actions,winner, + race,enemy_race] + return full_state \ No newline at end of file From 6c6caf4eaddd0955baf58233f04748fc64fdf276 Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 10 Nov 2017 13:36:53 -0700 Subject: [PATCH 08/23] updates as per @tewalds review --- pysc2/bin/replay_actions.py | 30 +++-- pysc2/lib/renderer_human.py | 4 - pysc2/replay_parsers/action_parser.py | 168 +++++++++++++------------- pysc2/replay_parsers/base_parser.py | 78 ++++++------ pysc2/replay_parsers/state_parser.py | 146 +++++----------------- 5 files changed, 172 insertions(+), 254 deletions(-) diff --git a/pysc2/bin/replay_actions.py b/pysc2/bin/replay_actions.py index 1e6c938c4..4b0f6cca9 100755 --- a/pysc2/bin/replay_actions.py +++ b/pysc2/bin/replay_actions.py @@ -51,8 +51,8 @@ flags.DEFINE_integer("step_mul", 8, "How many game steps per observation.") flags.DEFINE_string("replays", None, "Path to a directory of replays.") flags.DEFINE_string("parser", "pysc2.replay_parsers.base_parser.BaseParser", - "Which agent to run") -flags.DEFINE_string("data_dir", "C:/", + "Which parser to use in scrapping replay data") +flags.DEFINE_string("data_dir", None, "Path to directory to save replay data from replay parser") flags.DEFINE_integer("screen_resolution", 16, "Resolution for screen feature layers.") @@ -146,7 +146,7 @@ def run(self): self._print("Starting %s from player %s's perspective" % ( replay_name, player_id)) self.process_replay(controller, replay_data, map_data, - player_id,info,replay_name) + player_id, info, replay_name) else: self._print("Replay is invalid.") self.stats.parser.invalid_replays.add(replay_name) @@ -167,7 +167,8 @@ def _update_stage(self, stage): self.stats.update(stage) self.stats_queue.put(self.stats) - def process_replay(self, controller, replay_data, map_data, player_id,info,replay_name): + def process_replay(self, controller, replay_data, map_data, player_id, info, replay_name): + print(replay_name) """Process a single replay, updating the stats.""" self._update_stage("start_replay") controller.start_replay(sc_pb.RequestStartReplay( @@ -186,20 +187,25 @@ def process_replay(self, controller, replay_data, map_data, player_id,info,repla self.stats.parser.steps += 1 self._update_stage("observe") obs = controller.observe() - - #if parser.parse_step returns, whatever is returned is appended - #to a data list, and this data list is saved to a json file - #in the data_dir directory with filename = replay_name_player_id.json + # If parser.parse_step returns, whatever is returned is appended + # to a data list, and this data list is saved to a json file + # in the data_dir directory with filename = replay_name_player_id.json parsed_data = self.stats.parser.parse_step(obs,feat,info) if parsed_data: data.append(parsed_data) if obs.player_result: - #save scraped replay data to file at end of replay if parser returns + # Save scraped replay data to file at end of replay if parser returns + # and data_dir provided if data: - data_file = FLAGS.data_dir + replay_name + "_" + str(player_id) + '.json' - with open(data_file,'w') as outfile: - json.dump(data,outfile) + if FLAGS.data_dir: + stripped_replay_name = replay_name.split(".")[0] + data_file = os.path.join(FLAGS.data_dir, + stripped_replay_name + "_" + str(player_id) + '.json') + with open(data_file,'w') as outfile: + json.dump(data,outfile) + else: + print("Please provide a directory as data_dir to save scrapped data files") break self._update_stage("step") diff --git a/pysc2/lib/renderer_human.py b/pysc2/lib/renderer_human.py index 0598dbb57..d055536f3 100644 --- a/pysc2/lib/renderer_human.py +++ b/pysc2/lib/renderer_human.py @@ -41,10 +41,6 @@ from s2clientprotocol import sc2api_pb2 as sc_pb from s2clientprotocol import spatial_pb2 as sc_spatial -import ctypes - -ctypes.windll.user32.SetProcessDPIAware() - sw = stopwatch.sw render_lock = threading.Lock() # Serialize all window/render operations. diff --git a/pysc2/replay_parsers/action_parser.py b/pysc2/replay_parsers/action_parser.py index 47c68c8a5..154df3178 100644 --- a/pysc2/replay_parsers/action_parser.py +++ b/pysc2/replay_parsers/action_parser.py @@ -23,96 +23,96 @@ from pysc2.replay_parsers import base_parser class ActionParser(base_parser.BaseParser): - """Action statistics parser for replays.""" + """Action statistics parser for replays.""" - def __init__(self): - super(ActionParser, self).__init__() - self.camera_move = 0 - self.select_pt = 0 - self.select_rect = 0 - self.control_group = 0 - self.unit_ids = collections.defaultdict(int) - self.valid_abilities = collections.defaultdict(int) - self.made_abilities = collections.defaultdict(int) - self.valid_actions = collections.defaultdict(int) - self.made_actions = collections.defaultdict(int) + def __init__(self): + super(ActionParser, self).__init__() + self.camera_move = 0 + self.select_pt = 0 + self.select_rect = 0 + self.control_group = 0 + self.unit_ids = collections.defaultdict(int) + self.valid_abilities = collections.defaultdict(int) + self.made_abilities = collections.defaultdict(int) + self.valid_actions = collections.defaultdict(int) + self.made_actions = collections.defaultdict(int) - def merge(self, other): - """Merge another ReplayStats into this one.""" - def merge_dict(a, b): - for k, v in six.iteritems(b): - a[k] += v - super(ActionParser,self).merge(other) - self.camera_move += other.camera_move - self.select_pt += other.select_pt - self.select_rect += other.select_rect - self.control_group += other.control_group - merge_dict(self.unit_ids, other.unit_ids) - merge_dict(self.valid_abilities, other.valid_abilities) - merge_dict(self.made_abilities, other.made_abilities) - merge_dict(self.valid_actions, other.valid_actions) - merge_dict(self.made_actions, other.made_actions) + def merge(self, other): + """Merge another ReplayStats into this one.""" + def merge_dict(a, b): + for k, v in six.iteritems(b): + a[k] += v + super(ActionParser,self).merge(other) + self.camera_move += other.camera_move + self.select_pt += other.select_pt + self.select_rect += other.select_rect + self.control_group += other.control_group + merge_dict(self.unit_ids, other.unit_ids) + merge_dict(self.valid_abilities, other.valid_abilities) + merge_dict(self.made_abilities, other.made_abilities) + merge_dict(self.valid_actions, other.valid_actions) + merge_dict(self.made_actions, other.made_actions) - def valid_replay(self,info, ping): - """Make sure the replay isn't corrupt, and is worth looking at.""" - if (info.HasField("error") or - info.base_build != ping.base_build or # different game version - info.game_duration_loops < 1000 or - len(info.player_info) != 2): - # Probably corrupt, or just not interesting. - return False - for p in info.player_info: - if p.player_apm < 10 or p.player_mmr < 1000: - # Low APM = player just standing around. - # Low MMR = corrupt replay or player who is weak. - return False - return True + def valid_replay(self,info, ping): + """Make sure the replay isn't corrupt, and is worth looking at.""" + if (info.HasField("error") or + info.base_build != ping.base_build or # different game version + info.game_duration_loops < 1000 or + len(info.player_info) != 2): + # Probably corrupt, or just not interesting. + return False + for p in info.player_info: + if p.player_apm < 10 or p.player_mmr < 1000: + # Low APM = player just standing around. + # Low MMR = corrupt replay or player who is weak. + return False + return True - def __str__(self): - len_sorted_dict = lambda s: (len(s), self.sorted_dict_str(s)) - len_sorted_list = lambda s: (len(s), sorted(s)) - return "\n\n".join(( - "Replays: %s, Steps total: %s" % (self.replays, self.steps), - "Camera move: %s, Select pt: %s, Select rect: %s, Control group: %s" % ( - self.camera_move, self.select_pt, self.select_rect, - self.control_group), - "Maps: %s\n%s" % len_sorted_dict(self.maps), - "Races: %s\n%s" % len_sorted_dict(self.races), - "Unit ids: %s\n%s" % len_sorted_dict(self.unit_ids), - "Valid abilities: %s\n%s" % len_sorted_dict(self.valid_abilities), - "Made abilities: %s\n%s" % len_sorted_dict(self.made_abilities), - "Valid actions: %s\n%s" % len_sorted_dict(self.valid_actions), - "Made actions: %s\n%s" % len_sorted_dict(self.made_actions), - "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays), - "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), - )) + def __str__(self): + len_sorted_dict = lambda s: (len(s), self.sorted_dict_str(s)) + len_sorted_list = lambda s: (len(s), sorted(s)) + return "\n\n".join(( + "Replays: %s, Steps total: %s" % (self.replays, self.steps), + "Camera move: %s, Select pt: %s, Select rect: %s, Control group: %s" % ( + self.camera_move, self.select_pt, self.select_rect, + self.control_group), + "Maps: %s\n%s" % len_sorted_dict(self.maps), + "Races: %s\n%s" % len_sorted_dict(self.races), + "Unit ids: %s\n%s" % len_sorted_dict(self.unit_ids), + "Valid abilities: %s\n%s" % len_sorted_dict(self.valid_abilities), + "Made abilities: %s\n%s" % len_sorted_dict(self.made_abilities), + "Valid actions: %s\n%s" % len_sorted_dict(self.valid_actions), + "Made actions: %s\n%s" % len_sorted_dict(self.made_actions), + "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays), + "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), + )) - def parse_step(self,obs,feat,info): - for action in obs.actions: - act_fl = action.action_feature_layer - if act_fl.HasField("unit_command"): - self.made_abilities[ - act_fl.unit_command.ability_id] += 1 - if act_fl.HasField("camera_move"): - self.camera_move += 1 - if act_fl.HasField("unit_selection_point"): - self.select_pt += 1 - if act_fl.HasField("unit_selection_rect"): - self.select_rect += 1 - if action.action_ui.HasField("control_group"): - self.control_group += 1 + def parse_step(self,obs,feat,info): + for action in obs.actions: + act_fl = action.action_feature_layer + if act_fl.HasField("unit_command"): + self.made_abilities[ + act_fl.unit_command.ability_id] += 1 + if act_fl.HasField("camera_move"): + self.camera_move += 1 + if act_fl.HasField("unit_selection_point"): + self.select_pt += 1 + if act_fl.HasField("unit_selection_rect"): + self.select_rect += 1 + if action.action_ui.HasField("control_group"): + self.control_group += 1 - try: - func = feat.reverse_action(action).function - except ValueError: - func = -1 - self.made_actions[func] += 1 + try: + func = feat.reverse_action(action).function + except ValueError: + func = -1 + self.made_actions[func] += 1 - for valid in obs.observation.abilities: - self.valid_abilities[valid.ability_id] += 1 + for valid in obs.observation.abilities: + self.valid_abilities[valid.ability_id] += 1 - for u in obs.observation.raw_data.units: - self.unit_ids[u.unit_type] += 1 + for u in obs.observation.raw_data.units: + self.unit_ids[u.unit_type] += 1 - for ability_id in feat.available_actions(obs.observation): - self.valid_actions[ability_id] += 1 \ No newline at end of file + for ability_id in feat.available_actions(obs.observation): + self.valid_actions[ability_id] += 1 diff --git a/pysc2/replay_parsers/base_parser.py b/pysc2/replay_parsers/base_parser.py index 20e7676c4..d984f4838 100644 --- a/pysc2/replay_parsers/base_parser.py +++ b/pysc2/replay_parsers/base_parser.py @@ -17,48 +17,48 @@ import six class BaseParser(object): - """A base replay parser to write custom replay data scrappers.""" - def __init__(self): - self.replays = 0 - self.steps = 0 - self.maps = collections.defaultdict(int) - self.races = collections.defaultdict(int) - self.crashing_replays = set() - self.invalid_replays = set() + """A base replay parser to write custom replay data scrappers.""" + def __init__(self): + self.replays = 0 + self.steps = 0 + self.maps = collections.defaultdict(int) + self.races = collections.defaultdict(int) + self.crashing_replays = set() + self.invalid_replays = set() - def merge(self, other): - """Merge another ReplayStats into this one.""" + def merge(self, other): + """Merge another ReplayStats into this one.""" - def merge_dict(a, b): - for k, v in six.iteritems(b): - a[k] += v - self.replays += other.replays - self.steps += other.steps - merge_dict(self.maps, other.maps) - merge_dict(self.races, other.races) - self.crashing_replays |= other.crashing_replays - self.invalid_replays |= other.invalid_replays + def merge_dict(a, b): + for k, v in six.iteritems(b): + a[k] += v + self.replays += other.replays + self.steps += other.steps + merge_dict(self.maps, other.maps) + merge_dict(self.races, other.races) + self.crashing_replays |= other.crashing_replays + self.invalid_replays |= other.invalid_replays - def __str__(self): - len_sorted_dict = lambda s: (len(s), self.sorted_dict_str(s)) - len_sorted_list = lambda s: (len(s), sorted(s)) - return "\n\n".join(( - "Replays: %s, Steps total: %s" % (self.replays, self.steps), - "Maps: %s\n%s" % len_sorted_dict(self.maps), - "Races: %s\n%s" % len_sorted_dict(self.races), - "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays), - "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), - )) + def __str__(self): + len_sorted_dict = lambda s: (len(s), self.sorted_dict_str(s)) + len_sorted_list = lambda s: (len(s), sorted(s)) + return "\n\n".join(( + "Replays: %s, Steps total: %s" % (self.replays, self.steps), + "Maps: %s\n%s" % len_sorted_dict(self.maps), + "Races: %s\n%s" % len_sorted_dict(self.races), + "Crashing replays: %s\n%s" % len_sorted_list(self.crashing_replays), + "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), + )) - def valid_replay(self,info, ping): - #All replays are valid in the base parser - return True + def valid_replay(self,info, ping): + # All replays are valid in the base parser + return True - def parse_step(self,obs,feat,info): - #base parser doesn't directly parse any data, - #parse_step is a required function for parsers - pass + def parse_step(self,obs,feat,info): + # Base parser doesn't directly parse any data, + # parse_step is a required function for parsers + raise NotImplementedError() - def sorted_dict_str(self, d): - return "{%s}" % ", ".join("%s: %s" % (k, d[k]) - for k in sorted(d, key=d.get, reverse=True)) + def sorted_dict_str(self, d): + return "{%s}" % ", ".join("%s: %s" % (k, d[k]) + for k in sorted(d, key=d.get, reverse=True)) diff --git a/pysc2/replay_parsers/state_parser.py b/pysc2/replay_parsers/state_parser.py index 1db287e48..c112a3659 100644 --- a/pysc2/replay_parsers/state_parser.py +++ b/pysc2/replay_parsers/state_parser.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Action statistics parser for replays.""" +"""Example parser to collect some basic state data from replays. + The parser collects the General player information at each step, + along with the winning player_id of the replay""" from __future__ import absolute_import from __future__ import division @@ -23,118 +25,32 @@ from pysc2.replay_parsers import base_parser -from s2clientprotocol import sc2api_pb2 as sc_pb -from s2clientprotocol import common_pb2 as sc_common - -def calc_armies(screen): - friendly_army = [] - enemy_army = [] - unit_list = np.unique(screen[6]) - for unit in unit_list: - friendly_pixels = (screen[5] == 1) & (screen[6] == unit) - friendly_unit_count = sum(screen[11,friendly_pixels]) - #only append if count > 0 - if friendly_unit_count: - friendly_army.append([int(unit),friendly_unit_count]) - enemy_pixels = (screen[5] == 4) & (screen[6] == unit) - enemy_unit_count = sum(screen[11,enemy_pixels]) - if enemy_unit_count: - enemy_army.append([int(unit), enemy_unit_count]) - return friendly_army, enemy_army - -def update_minimap(minimap,screen): - #Update minimap data with screen details - #Identify which minimap squares are on screen - visible = minimap[1] == 1 - #TODO: need to devide screen into visible minimap, for now - #devide each quantity by number of visible minimap squares - total_visible = sum(visible.ravel()) - #power - minimap[4,visible] = (sum(screen[3].ravel())/ - (len(screen[3].ravel())*total_visible)) - #friendy army - friendly_units = screen[5] == 1 - #unit density - minimap[5,visible] = sum(screen[11,friendly_units])/total_visible - #Most common unit - if friendly_units.any() == True: - minimap[6,visible] = np.bincount(screen[6,friendly_units]).argmax() - else: - minimap[6,visible] = 0 - #Total HP + Shields - minimap[7,visible] = ((sum(screen[8,friendly_units]) + - sum(screen[10,friendly_units]))/total_visible) - #enemy army - enemy_units = screen[5] == 4 - #unit density - minimap[8,visible] = sum(screen[11,enemy_units])/total_visible - #main unit - if enemy_units.any() == True: - minimap[9,visible] = np.bincount(screen[6,enemy_units]).argmax() - else: - minimap[9,visible] = 0 - #Total HP + shields - minimap[10,visible] = ((sum(screen[8,enemy_units]) + - sum(screen[10,friendly_units]))/total_visible) - - return minimap - class StateParser(base_parser.BaseParser): - """Action statistics parser for replays.""" - def valid_replay(self,info, ping): - return True - """Make sure the replay isn't corrupt, and is worth looking at.""" - if (info.HasField("error") or - info.base_build != ping.base_build or # different game version - info.game_duration_loops < 1000 or - len(info.player_info) != 2): - # Probably corrupt, or just not interesting. - return False - for p in info.player_info: - if p.player_apm < 10 or p.player_mmr < 1000: - # Low APM = player just standing around. - # Low MMR = corrupt replay or player who is weak. - return False - return True - - def parse_step(self,obs,feat,info): - actions = [] - for action in obs.actions: - try: - full_act = feat.reverse_action(action) - func = full_act.function - args = full_act.arguments - except ValueError: - func = -1 - args = [] - - actions.append([func,args]) - - all_features = feat.transform_obs(obs.observation) - #remove elevation, viz and selected data from minimap - minimap_data = all_features['minimap'][2:6,:,:] - screen = all_features['screen'] - - mini_shape = minimap_data.shape - minimap = np.zeros(shape=(11,mini_shape[1],mini_shape[2]),dtype=np.int) - minimap[0:4,:,:] = minimap_data - extended_minimap = update_minimap(minimap,screen).tolist() - friendly_army, enemy_army = calc_armies(screen) - - if info.player_info[0].player_result.result == 'Victory': - winner = 1 - else: - winner = 2 - for player_id in [1, 2]: - race = sc_common.Race.Name(info.player_info[player_id-1].player_info.race_actual) - if player_id == 1: - enemy = 2 - else: - enemy = 1 - enemy_race = sc_common.Race.Name(info.player_info[enemy-1].player_info.race_actual) - - full_state = [info.map_name, extended_minimap, - friendly_army,enemy_army,all_features['player'].tolist(), - all_features['available_actions'].tolist(),actions,winner, - race,enemy_race] - return full_state \ No newline at end of file + """Example parser for collection General player information + from replays.""" + def valid_replay(self,info, ping): + """Make sure the replay isn't corrupt, and is worth looking at.""" + if (info.HasField("error") or + info.base_build != ping.base_build or # different game version + info.game_duration_loops < 1000 or + len(info.player_info) != 2): + # Probably corrupt, or just not interesting. + return False + for p in info.player_info: + if p.player_apm < 10 or p.player_mmr < 1000: + # Low APM = player just standing around. + # Low MMR = corrupt replay or player who is weak. + return False + return True + + def parse_step(self,obs,feat,info): + # Obtain feature layers from current step observations + all_features = feat.transform_obs(obs.observation) + player_resources = all_features['player'].tolist() + + if info.player_info[0].player_result.result == 'Victory': + winner = 1 + else: + winner = 2 + # Return current replay step data to be appended and save to file + return [player_resources,winner] \ No newline at end of file From bf4ad866ba47a4f95a7dada0cf9a5630225319d4 Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 10 Nov 2017 13:40:43 -0700 Subject: [PATCH 09/23] remove local flags workaround --- pysc2/bin/replay_actions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysc2/bin/replay_actions.py b/pysc2/bin/replay_actions.py index 4b0f6cca9..0604cf525 100755 --- a/pysc2/bin/replay_actions.py +++ b/pysc2/bin/replay_actions.py @@ -59,7 +59,6 @@ flags.DEFINE_integer("minimap_resolution", 16, "Resolution for minimap feature layers.") flags.mark_flag_as_required("replays") -FLAGS(sys.argv) interface = sc_pb.InterfaceOptions() interface.raw = True From e642f7264ede7f25cc3632685def55fed0be559c Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 10 Nov 2017 13:46:22 -0700 Subject: [PATCH 10/23] trailing line --- pysc2/replay_parsers/state_parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysc2/replay_parsers/state_parser.py b/pysc2/replay_parsers/state_parser.py index c112a3659..e4bfb767b 100644 --- a/pysc2/replay_parsers/state_parser.py +++ b/pysc2/replay_parsers/state_parser.py @@ -53,4 +53,5 @@ def parse_step(self,obs,feat,info): else: winner = 2 # Return current replay step data to be appended and save to file - return [player_resources,winner] \ No newline at end of file + return [player_resources,winner] + \ No newline at end of file From a25dff6dc28f96c2612e601134585e9fb2bf6cdf Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 10 Nov 2017 13:47:06 -0700 Subject: [PATCH 11/23] trailing line --- pysc2/replay_parsers/state_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysc2/replay_parsers/state_parser.py b/pysc2/replay_parsers/state_parser.py index e4bfb767b..e756fbb85 100644 --- a/pysc2/replay_parsers/state_parser.py +++ b/pysc2/replay_parsers/state_parser.py @@ -54,4 +54,3 @@ def parse_step(self,obs,feat,info): winner = 2 # Return current replay step data to be appended and save to file return [player_resources,winner] - \ No newline at end of file From 5dd050baef335c4fb49a6ea525d5380fc883dfef Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 10 Nov 2017 14:12:32 -0700 Subject: [PATCH 12/23] renamed state parser to player_info_parser --- README.md | 47 +++++++++++-------- ...{state_parser.py => player_info_parser.py} | 0 2 files changed, 27 insertions(+), 20 deletions(-) rename pysc2/replay_parsers/{state_parser.py => player_info_parser.py} (100%) diff --git a/README.md b/README.md index 68ae4386e..c571a5e6e 100644 --- a/README.md +++ b/README.md @@ -146,20 +146,6 @@ The left side is a basic rendering (which will likely be replaced by a proper rendering some day). The right side is the feature layers that the agent receives, with some coloring to make it more useful to us. -## Watch a replay - -Running the random agent and playing as a human save a replay by default. You -can watch that replay by running: - -```shell -$ python -m pysc2.bin.play --replay -``` - -This works for any replay as long as the map can be found by the game. - -The same controls work as for playing the game, so `F4` to exit, `pgup`/`pgdn` -to control the speed, etc. - ## List the maps [Maps](docs/maps.md) need to be configured before they're known to the @@ -190,11 +176,6 @@ configure your own, take a look [here](docs/maps.md). A replay lets you review what happened during a game. You can see the actions and observations that each player made as they played. -Blizzard is releasing a large number of anonymized 1v1 replays played on the -ladder. You can find instructions for how to get the -[replay files](https://github.com/Blizzard/s2client-proto#downloads) on their -site. You can also review your own replays. - Replays can be played back to get the observations and actions made during that game. The observations are rendered at the resolution you request, so may differ from what the human actually saw. Similarly the actions specify a point, which @@ -203,6 +184,32 @@ match in our observations, though they should be fairly similar. Replays are version dependent, so a 3.15 replay will fail in a 3.16 binary. +## Watch a replay + +Running the random agent and playing as a human will save a replay by default. You +can watch that replay by running: + +```shell +$ python -m pysc2.bin.play --replay +``` + +This works for any replay as long as the map can be found by the game. + +The same controls work as for playing the game, so `F4` to exit, `pgup`/`pgdn` +to control the speed, etc. + + + You can visualize the replays with the full game, or with `pysc2.bin.play`. Alternatively you can run `pysc2.bin.replay_actions` to process many replays -in parallel. +in parallel by supplying a replay directory. Each replay in the supplied directory +will be processed. + +```shell +$ python -m pysc2.bin.pl --replay +``` + +Blizzard is releasing a large number of anonymized 1v1 replays played on the +ladder. You can find instructions for how to get the +[replay files](https://github.com/Blizzard/s2client-proto#downloads) on their +site. You can also review your own replays. diff --git a/pysc2/replay_parsers/state_parser.py b/pysc2/replay_parsers/player_info_parser.py similarity index 100% rename from pysc2/replay_parsers/state_parser.py rename to pysc2/replay_parsers/player_info_parser.py From 6fa4d1ea36f458a829797ee1dad2b917803ab498 Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 10 Nov 2017 14:55:43 -0700 Subject: [PATCH 13/23] added documentation for replay parsing --- README.md | 31 +++++++++++++-- docs/environment.md | 21 +++++++++++ pysc2/bin/replay_actions.py | 3 +- pysc2/replay_parsers/state_parser.py | 56 ++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 pysc2/replay_parsers/state_parser.py diff --git a/README.md b/README.md index c571a5e6e..631f81443 100644 --- a/README.md +++ b/README.md @@ -198,18 +198,41 @@ This works for any replay as long as the map can be found by the game. The same controls work as for playing the game, so `F4` to exit, `pgup`/`pgdn` to control the speed, etc. - - You can visualize the replays with the full game, or with `pysc2.bin.play`. Alternatively you can run `pysc2.bin.replay_actions` to process many replays in parallel by supplying a replay directory. Each replay in the supplied directory will be processed. ```shell -$ python -m pysc2.bin.pl --replay +$ python -m pysc2.bin.replay_actions --replays +``` +The default number of instances to run in parallel is 1, but can be changed using +the `parallel` argument. + +```shell +$ python -m pysc2.bin.replay_actions --replays --parallel +``` + +## Parse a replay + +To collect data from one or more replays, a replay parser can be used. Two example +replay parsers can be found in the replay_parsers folder: + +* `action_parser`: Collects statistics about actions and general replay stats and prints to console +* `player_info_parser`: Collects General player info at each replay step and saves to file + +To run a specific replay parser, pass the parser as the `parser` argument. If the replay parser +returns data to be stored in a file, a directory must be supplied to the `data_dir` argument + +```shell +$ python -m pysc2.bin.replay_actions --replays --parser pysc2.replay_parsers.action_parser.ActionParser --data_dir ``` +Details on how to implement a custom replay parser can be found in the [here](docs/environment.md#replay-parsers). + +##Public Replays + Blizzard is releasing a large number of anonymized 1v1 replays played on the ladder. You can find instructions for how to get the [replay files](https://github.com/Blizzard/s2client-proto#downloads) on their -site. You can also review your own replays. +site. diff --git a/docs/environment.md b/docs/environment.md index d42352ce2..d82fd8c71 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -439,3 +439,24 @@ There are a couple basic agents. * `random_agent`: Just plays randomly, shows how to make valid moves. * `scripted_agent`: These are scripted for a single easy map. + +## Replay Parsers + +Custom replay parsers can be built to collect data from replay files. Two example +replay parsers can be found in the replay_parsers folder: + +* `action_parser`: Collects statistics about actions and general replay stats and prints to console +* `player_info_parser`: Collects General player info at each replay step and saves to file + +To build a custom replay parser, a class that inherits from the BaseParser needs to be defined. +The main method of the replay parser is the `parse_step` method. This function must take as arguments: +`obs`,`feat` and `info` which are the game observations, feature layers and replay information at a single +step in the replay, which is passed to the parser from `replay_actions` script for each step in the +replay file. This information is used to parse the desired data. If the `parse_step` function returns, +the returned value is appended to a list containing the parsed data for each step in the replay. Once the +replay is finsished, this list is saved to a data file in the supplied `data_dir` directory. +If no directory is supplied, the data is not saved to a file. + +The `valid_replay` method of the parent BaseParser class can be overridden to supply a custom +definition for valid replays (ie. filter out replays having players with small MMR). The `valid_replay` +method must take as arguments `info` and `ping` supplied from `replay_action` and return a boolean. \ No newline at end of file diff --git a/pysc2/bin/replay_actions.py b/pysc2/bin/replay_actions.py index 0604cf525..a658b4a75 100755 --- a/pysc2/bin/replay_actions.py +++ b/pysc2/bin/replay_actions.py @@ -50,7 +50,7 @@ flags.DEFINE_integer("parallel", 1, "How many instances to run in parallel.") flags.DEFINE_integer("step_mul", 8, "How many game steps per observation.") flags.DEFINE_string("replays", None, "Path to a directory of replays.") -flags.DEFINE_string("parser", "pysc2.replay_parsers.base_parser.BaseParser", +flags.DEFINE_string("parser", "pysc2.replay_parsers.action_parser.ActionParser", "Which parser to use in scrapping replay data") flags.DEFINE_string("data_dir", None, "Path to directory to save replay data from replay parser") @@ -59,6 +59,7 @@ flags.DEFINE_integer("minimap_resolution", 16, "Resolution for minimap feature layers.") flags.mark_flag_as_required("replays") +FLAGS(sys.argv) interface = sc_pb.InterfaceOptions() interface.raw = True diff --git a/pysc2/replay_parsers/state_parser.py b/pysc2/replay_parsers/state_parser.py new file mode 100644 index 000000000..e756fbb85 --- /dev/null +++ b/pysc2/replay_parsers/state_parser.py @@ -0,0 +1,56 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Example parser to collect some basic state data from replays. + The parser collects the General player information at each step, + along with the winning player_id of the replay""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import collections +import six +import numpy as np + +from pysc2.replay_parsers import base_parser + +class StateParser(base_parser.BaseParser): + """Example parser for collection General player information + from replays.""" + def valid_replay(self,info, ping): + """Make sure the replay isn't corrupt, and is worth looking at.""" + if (info.HasField("error") or + info.base_build != ping.base_build or # different game version + info.game_duration_loops < 1000 or + len(info.player_info) != 2): + # Probably corrupt, or just not interesting. + return False + for p in info.player_info: + if p.player_apm < 10 or p.player_mmr < 1000: + # Low APM = player just standing around. + # Low MMR = corrupt replay or player who is weak. + return False + return True + + def parse_step(self,obs,feat,info): + # Obtain feature layers from current step observations + all_features = feat.transform_obs(obs.observation) + player_resources = all_features['player'].tolist() + + if info.player_info[0].player_result.result == 'Victory': + winner = 1 + else: + winner = 2 + # Return current replay step data to be appended and save to file + return [player_resources,winner] From 9969d86c176ef81f025ed698e42d542b078b091e Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 10 Nov 2017 14:56:21 -0700 Subject: [PATCH 14/23] remove state-parser example --- pysc2/replay_parsers/state_parser.py | 56 ---------------------------- 1 file changed, 56 deletions(-) delete mode 100644 pysc2/replay_parsers/state_parser.py diff --git a/pysc2/replay_parsers/state_parser.py b/pysc2/replay_parsers/state_parser.py deleted file mode 100644 index e756fbb85..000000000 --- a/pysc2/replay_parsers/state_parser.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2017 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS-IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Example parser to collect some basic state data from replays. - The parser collects the General player information at each step, - along with the winning player_id of the replay""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import collections -import six -import numpy as np - -from pysc2.replay_parsers import base_parser - -class StateParser(base_parser.BaseParser): - """Example parser for collection General player information - from replays.""" - def valid_replay(self,info, ping): - """Make sure the replay isn't corrupt, and is worth looking at.""" - if (info.HasField("error") or - info.base_build != ping.base_build or # different game version - info.game_duration_loops < 1000 or - len(info.player_info) != 2): - # Probably corrupt, or just not interesting. - return False - for p in info.player_info: - if p.player_apm < 10 or p.player_mmr < 1000: - # Low APM = player just standing around. - # Low MMR = corrupt replay or player who is weak. - return False - return True - - def parse_step(self,obs,feat,info): - # Obtain feature layers from current step observations - all_features = feat.transform_obs(obs.observation) - player_resources = all_features['player'].tolist() - - if info.player_info[0].player_result.result == 'Victory': - winner = 1 - else: - winner = 2 - # Return current replay step data to be appended and save to file - return [player_resources,winner] From b65ed0d09860d045267e277149a7b5b3d4a9f06d Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 10 Nov 2017 15:01:18 -0700 Subject: [PATCH 15/23] doc edits and typo fixes --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 631f81443..ac67b05c2 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ $ python -m pysc2.bin.replay_actions --replays --pars Details on how to implement a custom replay parser can be found in the [here](docs/environment.md#replay-parsers). -##Public Replays +## Public Replays Blizzard is releasing a large number of anonymized 1v1 replays played on the ladder. You can find instructions for how to get the From 677ec46aae3aa7d0c3b1d09ed34fd15c4d59f6fe Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 10 Nov 2017 15:21:01 -0700 Subject: [PATCH 16/23] rename StateParser to PlayerInfoParser --- docs/environment.md | 4 ++-- pysc2/replay_parsers/player_info_parser.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/environment.md b/docs/environment.md index d82fd8c71..c12c51dab 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -448,13 +448,13 @@ replay parsers can be found in the replay_parsers folder: * `action_parser`: Collects statistics about actions and general replay stats and prints to console * `player_info_parser`: Collects General player info at each replay step and saves to file -To build a custom replay parser, a class that inherits from the BaseParser needs to be defined. +To build a custom replay parser, a class that inherits from BaseParser needs to be defined. The main method of the replay parser is the `parse_step` method. This function must take as arguments: `obs`,`feat` and `info` which are the game observations, feature layers and replay information at a single step in the replay, which is passed to the parser from `replay_actions` script for each step in the replay file. This information is used to parse the desired data. If the `parse_step` function returns, the returned value is appended to a list containing the parsed data for each step in the replay. Once the -replay is finsished, this list is saved to a data file in the supplied `data_dir` directory. +replay is finished, this list is saved to a data file in the supplied `data_dir` directory. If no directory is supplied, the data is not saved to a file. The `valid_replay` method of the parent BaseParser class can be overridden to supply a custom diff --git a/pysc2/replay_parsers/player_info_parser.py b/pysc2/replay_parsers/player_info_parser.py index e756fbb85..d3e377219 100644 --- a/pysc2/replay_parsers/player_info_parser.py +++ b/pysc2/replay_parsers/player_info_parser.py @@ -25,10 +25,11 @@ from pysc2.replay_parsers import base_parser -class StateParser(base_parser.BaseParser): +class PlayerInfoParser(base_parser.BaseParser): """Example parser for collection General player information from replays.""" def valid_replay(self,info, ping): + return True """Make sure the replay isn't corrupt, and is worth looking at.""" if (info.HasField("error") or info.base_build != ping.base_build or # different game version From e9e49241ea10a8e790ac081897df5adbc892cf5d Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Sun, 12 Nov 2017 13:31:22 -0700 Subject: [PATCH 17/23] updated replay_actions to process_replays and documentation references --- README.md | 8 ++++---- docs/environment.md | 4 ++-- pysc2/bin/{replay_actions.py => process_replays.py} | 0 3 files changed, 6 insertions(+), 6 deletions(-) rename pysc2/bin/{replay_actions.py => process_replays.py} (100%) mode change 100755 => 100644 diff --git a/README.md b/README.md index ac67b05c2..7c00010c7 100644 --- a/README.md +++ b/README.md @@ -199,18 +199,18 @@ The same controls work as for playing the game, so `F4` to exit, `pgup`/`pgdn` to control the speed, etc. You can visualize the replays with the full game, or with `pysc2.bin.play`. -Alternatively you can run `pysc2.bin.replay_actions` to process many replays +Alternatively you can run `pysc2.bin.process_replays` to process many replays in parallel by supplying a replay directory. Each replay in the supplied directory will be processed. ```shell -$ python -m pysc2.bin.replay_actions --replays +$ python -m pysc2.bin.process_replays --replays ``` The default number of instances to run in parallel is 1, but can be changed using the `parallel` argument. ```shell -$ python -m pysc2.bin.replay_actions --replays --parallel +$ python -m pysc2.bin.process_replays --replays --parallel ``` ## Parse a replay @@ -225,7 +225,7 @@ To run a specific replay parser, pass the parser as the `parser` argument. If th returns data to be stored in a file, a directory must be supplied to the `data_dir` argument ```shell -$ python -m pysc2.bin.replay_actions --replays --parser pysc2.replay_parsers.action_parser.ActionParser --data_dir +$ python -m pysc2.bin.process_replays --replays --parser pysc2.replay_parsers.action_parser.ActionParser --data_dir ``` Details on how to implement a custom replay parser can be found in the [here](docs/environment.md#replay-parsers). diff --git a/docs/environment.md b/docs/environment.md index c12c51dab..a8bb13b48 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -451,7 +451,7 @@ replay parsers can be found in the replay_parsers folder: To build a custom replay parser, a class that inherits from BaseParser needs to be defined. The main method of the replay parser is the `parse_step` method. This function must take as arguments: `obs`,`feat` and `info` which are the game observations, feature layers and replay information at a single -step in the replay, which is passed to the parser from `replay_actions` script for each step in the +step in the replay, which is passed to the parser from `process_replays` script for each step in the replay file. This information is used to parse the desired data. If the `parse_step` function returns, the returned value is appended to a list containing the parsed data for each step in the replay. Once the replay is finished, this list is saved to a data file in the supplied `data_dir` directory. @@ -459,4 +459,4 @@ If no directory is supplied, the data is not saved to a file. The `valid_replay` method of the parent BaseParser class can be overridden to supply a custom definition for valid replays (ie. filter out replays having players with small MMR). The `valid_replay` -method must take as arguments `info` and `ping` supplied from `replay_action` and return a boolean. \ No newline at end of file +method must take as arguments `info` and `ping` supplied from `process_replays` and return a boolean. \ No newline at end of file diff --git a/pysc2/bin/replay_actions.py b/pysc2/bin/process_replays.py old mode 100755 new mode 100644 similarity index 100% rename from pysc2/bin/replay_actions.py rename to pysc2/bin/process_replays.py From 9b62b4785c961b21512a95cdc3630cda1f94ef2a Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Sun, 12 Nov 2017 13:32:12 -0700 Subject: [PATCH 18/23] fix typo --- pysc2/replay_parsers/player_info_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysc2/replay_parsers/player_info_parser.py b/pysc2/replay_parsers/player_info_parser.py index d3e377219..d097c5572 100644 --- a/pysc2/replay_parsers/player_info_parser.py +++ b/pysc2/replay_parsers/player_info_parser.py @@ -29,7 +29,6 @@ class PlayerInfoParser(base_parser.BaseParser): """Example parser for collection General player information from replays.""" def valid_replay(self,info, ping): - return True """Make sure the replay isn't corrupt, and is worth looking at.""" if (info.HasField("error") or info.base_build != ping.base_build or # different game version From ea30f5512ad24985e899376d7ac9de65ca620cfb Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 17 Nov 2017 10:42:09 -0700 Subject: [PATCH 19/23] building out unit tests for replay parsing --- pysc2/bin/process_replays.py | 2 +- pysc2/replay_parsers/action_parser.py | 4 +- pysc2/replay_parsers/base_parser.py | 4 +- pysc2/replay_parsers/player_info_parser.py | 4 +- pysc2/tests/replay_parser_test.py | 47 +++++++++++++++++++++ pysc2/tests/test_replay.SC2Replay | Bin 0 -> 17436 bytes 6 files changed, 54 insertions(+), 7 deletions(-) create mode 100644 pysc2/tests/replay_parser_test.py create mode 100644 pysc2/tests/test_replay.SC2Replay diff --git a/pysc2/bin/process_replays.py b/pysc2/bin/process_replays.py index a658b4a75..b6f38ac2d 100644 --- a/pysc2/bin/process_replays.py +++ b/pysc2/bin/process_replays.py @@ -250,7 +250,7 @@ def replay_queue_filler(replay_queue, replay_list): def main(unused_argv): - """Dump stats about all the actions that are in use in a set of replays.""" + """Collect data from a set of replays using supplied parser.""" run_config = run_configs.get() parser_module, parser_name = FLAGS.parser.rsplit(".", 1) diff --git a/pysc2/replay_parsers/action_parser.py b/pysc2/replay_parsers/action_parser.py index 154df3178..4c0e869f9 100644 --- a/pysc2/replay_parsers/action_parser.py +++ b/pysc2/replay_parsers/action_parser.py @@ -53,7 +53,7 @@ def merge_dict(a, b): merge_dict(self.valid_actions, other.valid_actions) merge_dict(self.made_actions, other.made_actions) - def valid_replay(self,info, ping): + def valid_replay(self, info, ping): """Make sure the replay isn't corrupt, and is worth looking at.""" if (info.HasField("error") or info.base_build != ping.base_build or # different game version @@ -87,7 +87,7 @@ def __str__(self): "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), )) - def parse_step(self,obs,feat,info): + def parse_step(self, obs, feat, info): for action in obs.actions: act_fl = action.action_feature_layer if act_fl.HasField("unit_command"): diff --git a/pysc2/replay_parsers/base_parser.py b/pysc2/replay_parsers/base_parser.py index d984f4838..c6a04f6ae 100644 --- a/pysc2/replay_parsers/base_parser.py +++ b/pysc2/replay_parsers/base_parser.py @@ -50,11 +50,11 @@ def __str__(self): "Invalid replays: %s\n%s" % len_sorted_list(self.invalid_replays), )) - def valid_replay(self,info, ping): + def valid_replay(self, info, ping): # All replays are valid in the base parser return True - def parse_step(self,obs,feat,info): + def parse_step(self, obs, feat, info): # Base parser doesn't directly parse any data, # parse_step is a required function for parsers raise NotImplementedError() diff --git a/pysc2/replay_parsers/player_info_parser.py b/pysc2/replay_parsers/player_info_parser.py index d097c5572..a2e048e94 100644 --- a/pysc2/replay_parsers/player_info_parser.py +++ b/pysc2/replay_parsers/player_info_parser.py @@ -28,7 +28,7 @@ class PlayerInfoParser(base_parser.BaseParser): """Example parser for collection General player information from replays.""" - def valid_replay(self,info, ping): + def valid_replay(self, info, ping): """Make sure the replay isn't corrupt, and is worth looking at.""" if (info.HasField("error") or info.base_build != ping.base_build or # different game version @@ -43,7 +43,7 @@ def valid_replay(self,info, ping): return False return True - def parse_step(self,obs,feat,info): + def parse_step(self, obs, feat, info): # Obtain feature layers from current step observations all_features = feat.transform_obs(obs.observation) player_resources = all_features['player'].tolist() diff --git a/pysc2/tests/replay_parser_test.py b/pysc2/tests/replay_parser_test.py new file mode 100644 index 000000000..698b467f0 --- /dev/null +++ b/pysc2/tests/replay_parser_test.py @@ -0,0 +1,47 @@ +#!/usr/bin/python +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Process a test replay using the BaseParser replay parser""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from pysc2 import run_configs +from pysc2.replay_parsers import base_parser +from pysc2.tests import utils + +from absl.testing import absltest as basetest + + +class TestBaseParser(utils.TestCase): + + def boolean_valid_replay(self): + + + def test_random_agent(self): + steps = 100 + step_mul = 50 + with sc2_env.SC2Env( + map_name="Simple64", + step_mul=step_mul, + game_steps_per_episode=steps * step_mul) as env: + agent = random_agent.RandomAgent() + run_loop.run_loop([agent], env, steps) + + self.assertEqual(agent.steps, steps) + + +if __name__ == "__main__": + basetest.main() diff --git a/pysc2/tests/test_replay.SC2Replay b/pysc2/tests/test_replay.SC2Replay new file mode 100644 index 0000000000000000000000000000000000000000..ddc6d0c74ae2cb1b1b6415f50ff8a5100ac7c230 GIT binary patch literal 17436 zcmeIZRa9KfzCF5t#wEBk5@;HiAdO2JcbDKU-8h5<4Z+>rgKKby;O-7V0|b&_fdmhc zKRJ7!JHB(qy$|>0zT7=~RM)R;RgYO!T~%vVscC9a1E>H1fB@k6gNIHGpmOSXT6)NM zSlW5gD=E=?*t$7c`cU)oqN3vg*r=%37$9tB3~W>^?70+79PGYCOb|8*1H1kW3Kj+m z4mv6-D*ajuLV-1KN`Y#=HM8$MG8FG`R)YUZdZ^-mfDq$<1pgEG4+H;U;QvDg)HJog za}V{=zySaaKnG|f9_*(8K>rX&_~8DDzj^**U+iCg*uVY%`v3SFh4QcX|Nhqax54_~ z@o)f8D5P5!gIC3#Ry?Eg^W*C#lBG=?RfXP)&ZSnV$9+Bi@3Q~x-wIx z8zDq^r3{&M41F+7U;rTS`sUA-o}Qjo8VCzR4UjC~-hrYJ1_B^dI1V5UN0H(I?zv=? zCDh)~;?7KDnK7wB{okTA^3JAw;K^H} zG0bRoy5BKCyVYXoq%Lc=IGTZ=1D<#HvQSVbBvhJQMbXHf^OQzG(>A${*N!geGFFS7 zM4mjf{i@v2-Y;OKRK})CsWRN#4X%7&Ld&YzB(5mAohrT?cx}T_>b~h8OhIYGP9J>6 zU`IW0Lxzfl+bUy2>dqJa7)Lao|G5ws18r3JS%2C;}oXWzA zE>#uDOX@z9%9^Hf+m*~V5E?Qjx?9j@fm-o6LI@}0-uIeQ9me_(mhnHCDgc1s|72P4 z|AyrsNNi2?M>?ce)H)8%GV)h&z*jyfM;E}#f)&^m{G}M>2*8-Y*O{-vuPorjivs|R zI6fGFs!89-ie?C}qNfQA(E`i%;bXufS(RYP@PvBXp(AI}(|bj2se$DM4eg5G3X|A% zHXA!exZ~kWQOuTI65eYE#rzDY^{=heOmT}@5&KW^g1<~ZEt+YUUy zT^Y6}3G0<$F+;Ho902GFmL=r#ODrj6PBqdrJ(%FBux#uHR2AFNZotYsQ`KGje2?dB$jb9hSL6C!JcS{oXbgFX)glo>1`sXD zolu@01MDyuOI1~rh)B1;bSY^Xqa(~qICnX{B%zC~z0&(JhS@{{Z3#xb<-*G9*wROh zG}P3PdClHx^a5sT3`I)}P(!~=ooAlmINpn0jtO~%lAn4|#gt?&oc5D1?$#vIblR}Q z;%^LLs&dl#k~q@q>Kuwg+>}ZcBsk;IKLlv^>HfrYKyq0X2?mE&0{^_pGL{%1qic@|3 zyic`Jd^4H!-$(Pvm+*LRScf8G-3Tf9-R9ClHG|ue1SGWhpxOX0okCI~OK%{ypHa-T z)K;0^6rhTkLCU_Ob&~XM11S|ERQ#FRVr_3Ze!wlTH!Sy2cYUdXwwkSDig-z|FB>xH z4F?C}r+dqbkNPaQKN{OKLksx%dGJGMXT)Yr@3^hA2as>C+Fm-oc=qy+0k_bHgH41Y zGNuO!5ZVBnvhrj%+yB)s89cs=N>}`@*X-rup3)R|?o0UrkNpSJ#SA+S>%!L$O#c`L zCH`d^hg0@rnH=i>VtSw+SZgBE(BcCl7UB2+O2HfegW+T3XeG``sY#Vr7R*up9oixA zzeEq~EK8^^Q1L4j%Pb%kiIR?N=99|isW6y{3iB($j5J=<3uf>PT51F_mu~k#*V>63xNKT`zjiws<{{8dpZ0 zUVVP)y}f(ii2Y}iA4tKoxTXi|v)3=H4{ za}G!t8X_+a3{dH@Wi0Qj#$0P;r_nTBfd5=voN zY={i}Z{h zhdWl(H1yFRLS1OGZYz@slTlTDuu^=o#iBER*~2WYL~pHD$T$4hcOkKg{lGw)wJ3>4 z1yoQ_Fg8(e#Ka>HR;5O;2fc1x04X_?MzSfUvl~RpX#rvSqP{1Y&F_4S2hu%S70Pn< z#{YDdetmtSPxR_$usvRjSD26By7ZZ3p!BQl?;N73CgUe{dDtZlVU$MIB^Aid&b5-a z>+J0eHb$93=moh*sVt%3MPg=ud%F-R$K7u=GQ%-Ej}KpT)O3$j-H6tnmIm!25#4XqHUcDD{RyZA;f_s9eW%+=!zq8$j-e%cgD-@zGS5c3>PQQWEL=VcD6RouUM z#qFV0MWN$?5NczeKGYz`-t{Pcl_R_B*?uZ7{o?kV3oo6Yxq*`+A~ah@?c#(}-gbZ~ z$#l#PG}!kIA-9|KZ3NMGS|Awf$OhAlE*~Ly z+n`ICr4B2|MIh_&)(LS;q6M?GfGkn@+H|8v(9k7wt?j ztDGJsI|1@$mNVC_}NBTO3D|P^Zisp4TM#vNPj^ zYOVJ#qJ1tn=Td*}D^uedILt)8SV+^I91?zCTHjNDZ!h-)yp@hOq;^rLAwtCJ5-+oA zp6{5k<}hs4QcAB}MA4ui8&tfBrtyKHxn|Jmh{i}t`aRYO2CTZ|(RT|%@>Nf2kh(IB z*{}MEAozLE#AUWqVuJyH=)ozym8MDtwre>?hi0pt-A!Uwu!@v^sRK0qtHAoATpm}| zfjCBr2!e2bEHx{$HGQst45f#H#(W5EUG_K7$I>;uj=`~?hxFDtRDd?q2hm4r7m-Tg zPO|su2OCA7sfxh~`&s*wI94fX>!rydlpEF#H7`RLiomkW)jBw89C*}ZZAu8`KHd5n z&SJNMjYj9BM_XL`6aB=#sxf#@hP>AO24lz!Zvz(-E9Yu3yZ9IfgVD*lii?c(QVf%w z9V=JAAv`VJZX56CS-R`}me=X0BfqZiof*yP^rGv=yfvQ%ubx!~7SZP;6}4?d_w>Q2 z8`50DPD4qc$ZSx_Twz>@wM_)GEwMIpCShIJdIJ+SYGHvh;OtP3Y^ipe8ySbv48W7E ziHX&KAqHOh3Vj2Wrb;p>%*Zn^nBD@_zy&O&`=h@3e%fmb|F`b=O|P1H$imu}z7dn4 z5J~6H=kXqeWXs5nw%K=9dEw3HptgnU7X?DgI6*|2(eV}}@f=pD~$;e)WW_`*qADb=1Y;jP!=-F8etn>iJjmN5H%s;ZsSyU5cc(G;#{4 zJ!Ifh8?g~0AKeBQtXy4XA_q#1dK$2rzT?`$$<*@1k$FhEAT^mCBwbLU3yDS{(aIF( zKHRwULQu=2nk)IUCWb~sbs>;YLN*jEHlPBPK9LlgI2s}=8yqRkiUpTtBO_A8DU@SX zl!dTI5=keKMkP3lmzl?^7`S#`tGvev1xjS4=p&;POL3xAk_}3*IBJR&hN;ShsBqW(1LG@vwp{3PMTt7C~h?#Cj9k=YX z{bVNE7l>~%IlpK>GndTo)k`w>om#rF8;UP2@d!r$*7*T#eFZJ;>UokgRxNLzR%}Vn zla@S2^L=g$ZmBb03aZdCI~J*_+B3Q$NZGq))uiV5k3Yg@Tm0u(x4_OU8%Fy}P{%|# zR)Gcns0Osx8ivP$BE18>cgjMnQ&SDd=IhRljNo@jyMsIyQj17obU7nI4%x-8X!ov9 zF_P3QZ5+5K2#;yeH|V%qD^bJvRM0%DW*PjxZTI7@qH8x$#t z)hUw^77THMXx3JQ%Q#X@Ec9B-Ohmdt(9EjTBH_K{JXmo_ZQ@XWDKO{7<&pdHt3u0O zA%5<*X$(L2L6Bu08k%}|5(qXTIEE@bwMG+CT6APpF^iqzTp&`Gs4)(wApt?1Lz5g6 zHUsjjw|nlJ&wtRI-h_O8k^dxN*rjo6*yV`sl_-)%Jr*w!nlX#%xy#xwyGV5A=Y~B#O4ByO+ND_sX@#*>4Bl1FG>>F zsb5(T7h;z^7C26O!IzBhNr)X63!h_vkxFY~$Dzr97F87>V&ppnhSw(qNhN`I#*>*j z+oN77S*y<_YHpnWjE=oZtS+q?wtSH+2JWP3?RgHJ>HK}NllTIAv&x2`P&A6-aBv(9 z9|=Y&#UtuJP|o}Dfkl8OOS5q}aDf~D#Yt(g|3Fzf+5m^XOIFOi;JFHP+x1;vWiHy= z5Ir{fs~Q>sqOd%<6Q#LoXtt2*uD9bM#HS@usrhNe7k#@#YdJ5--Hd=;^Q#C=TTu#} z%+Oe|XAO)pys(Oma1i&`Wh8F3C90ntgm~5{rHkSOduT2z=ow~Ho=%TL=^-@8g1^=-XDzcftGYEeuV5@+vy~ccO`qCXiw}+>7#7z8kO+yBQQM$+B^rFk|+D}5cc>$05^nPS8kE*HR z_I;@bk8*+R+(KLyS@H}bUh+HJP{CUpso3vy`E2FL=h7MikXP(6Sznx6g_F@M7~O|J z(jhI>6(3!-f}`;kP@>j{HtH@Sdq0k$f>7m1v0cu^mej~%C2>5PiEzV__e-{Lct!wDrW|oF1j}Az3QYgF z-I$l?J08#Rx3xCc)+GdSGs#pbcM77ptLv7wJN>KpB9UYUtCVuLrDW+7@9Wg$wOP_~ zX_SP_(lS;pQ!Vo@W)Y?e`E>33%W2K|kq!E_kF=t5Dw`5j6MZhm`d=69$ZAPDe$dJV z6?mwG?*e9i`zcy$Jcu=gAHSgVMl>`PbCRg2mnZct+Vn|y3Q#_+_9*phJv-NzKv6s( zx4@Je!1>utb9_YZ3*61%`pvg5JkoqL|$oiDpbeD9jvW&(K>fh_f`+Mav-JyofEtx&e=-6sq^w*Duad*-C7&V?}KE zRoCjDsQa+HqL|nHtsCITMQIQzWHU)S`| z;kcrY9M1Fc)4gk?*(xZT&AP`X=_^Fl&nyn9(rs^pV=f3s(wrGjHyiB8A(OaeWM)&A z9prqJr7P?_+uD@;w1$4W1S=m8vy6PnRIBh3lg+TWue@;?^~Lg|*_nSWrm!-s;J*(# zI+8V$!TS*A%$vg>V$#J~vW_1~;hLGj)k2{7*+AW<4m`b4##M1V3c)zt4B;P}5DBIA z)hWvsoyH`6ZjerP=J@$Xwgbz6WS$-enLS)awB#Db#h7J+#aF}NPM_gcn3j0THVtFh zr-vxxFcCFPO$#`$z3;6GEtYeiMqR{wszp#Sp{f@j-Nn%lX4lt45Tzb_pimAQi7Zxt zokcwhYOjr3`Pb|b0#hVI~7!i)VI~ZtBMQ10W zW+MTetL!g`NzXn}e*Zm$*Jp$rL?nmpT)yVsg3zUY2DB^br_LN>RqcA<37ZcZq@E~8 za(G;qVsl9QKf@2uUklAYUVQ}G`*fe#eKhTnq_AtE9jR8>@XVs-#MAqw`_W|3%);jA zfz$E}|5|;FutoitbT2i+^DH9Rg4F9rzF1RpCq?`8;+Hdq$mm6S*c3J;#sTG zzD$ITZdiWnA8kg)YflC87DC*o%`#i`Q54C)nx9Ej$=VSXVDON-bMttTy~0N#1xdxY z?Ce&>0=p1r3_SrqIcdrIR0e0Zq;$oREjoVF=qoW$>>s3|5r!M9;4CXrB?*>Lkvpuav zvq)6HBf%~4(aYJZcsXhfvDDPaSZfRctM)dr)Dh;j1QfhV?UPHB9PO2uyxxY_G5yb< zC_WmGu3#pz@?`DPWMyYGAPhHT+==F!uiq9E8v?TU@m+M6E<%iJ}T2RUxM~-^2 zwMg)5tByt6U$}j^pG^zE7vUw|T2d;0k@YU&GAU;iT+MS||77eIn&UotYt9V}v;Ha?js4b|^e-M9uD}lTj$)28l zB20j4lVFC^P$jhKRwgo`6D{K<;aJ=(yQyG7Oa*y~#a!89F|wJmdHv4*42oB#t1Elm zI8!=j2dPw$tjY21hc z^C*&UUcr5BOv()@m}8=Xj$POg>kMIFV8tP7=C^N#ZjauTQXc$9_rE&na>+=6H%jqX zPz4TA6q(C~Z2QMI3_GOlx`tv(Pi3%%e^lV%l}jj#(fYX=whCU(Ux90E9Pv4OmPPlp zM&ZSnm@doZAIaObR4TO8J1L;fO4~LeM7XVu8+bGG>n+=nk4#2Hi&yK`9mWaTeKUwz zlexA>3_0AGER%TAKej6EX2GSt;rN!Bgf-%Ypl%7m-T5$gfXeWhx|Jt)ocMl63!Fk)n{0YWj z;ZluJx02tI_ouwp(Sud``0T7TilLb*HgtFk76~Qa2m-k!GE%lG(TsrP0lt_S?H*HW zv976>A1)Nt%vVG&p0W2*9Q(g{Gk)K(mrC)echhJRWc2ph;ZDd#pk1P)R+ozm&Ua8&6q^S`wG4z}SC6dvI0L)<;Qw8rZT zv)T-b3<_%SXf%BJS?FEfuN$Yo8xmxI1yXDoQ=#ygwFSRdI%Fwd@>2 z#uLi_xTmQmi8OFr(twBvq@yGCUO>5~?p^5Uka0z@%P zO!YD0-DY=kc)98N99(c`m?_)##GigQTYg6_sr2Hig#J*weYFLcy^OBV$*5(PIA9{4 zm(awPJ^Rt)2-*lZz_UmgwRgw=QgeJ5fIESEs$%PC#C%Xat?t?R&SVOW?#%So^aYdM zZjXt(MaWNGss27T0Dh1uDn`z(SS29{Kq!`t=Wfy|AuRu01_r zDizE}ItLj8gCh6<&)~e3SwrIa$H26jlI|Bmx61C4mE%cIMv)Qc2&&ZE>I<|NMkpb z*o4LhGbzF16);uce%D>%QJH?JHf5Yw@6Q65%zXznwjH2!O4WPbio>MH5 zZ3D|^cs1P+e*Sn{M0>hUaC{(_ftpujmMjst~c6?7cSE3aL{( zWWZ3SE7u(9KiBMpFodv&a%!lakJYOC@eyKa1SHq; z|Hi!fYH7eUF=va#_CI+FJg8U;WsfLgMn+LBxL4FsOOuvdK<7A zGjz_SH)+rjcR1CQkwvLQlg_1hPhewTuDSwxF9_<(cG`nh|D<1suzwDiI6hp*g=tKYI*5#inp7us?a8Px(rIXtD>41L?jBWj!9C6|$+*1_l@)zo zYyfJP2#p-mQZk#NSWJBLR3QjW2%oiWEAr_)wiZJ~LCyG^h!j&QeHoLePeeRl^M6N! zA+}LLCsyO4WzQV~Sl$-AdMVefqPC#^iRLjQX`k`mf- zTU&&Mg$;giF3mj;N-Hp*JhD$FqDdM=10_gO`mr4YV^}rq+x|B4o z3T0#{G(Gq((BaN7d!HYI9hnCh+ih8b#E3U0AkfEQ-qg`u`2kw1?8u_}1DHwV6Z(95 z*-VD!Sp4{7nYUXXtsTD!_(}Tln>wQP#zivxsdFUW=R0`m>4xDdUM%Y3f3b1m`m)-k zoBhI>>DAgsX^EV*>^5PK>2E_b^oQZ`ajnNF2R+nZdx8=^tEKEB^*j9+9$}lvd*k0d zX&l_nb=dL2Vf*CW`3oX-TCI|idEc0~*0n_{OmUK@PB}Y|KZmoN|9l_TRQc|ghWh}5 zxPs-699dZFZvUr6oxr=k&yX@9%WT)7Y0Us*C)6M79ETU$cfUaKS+gAG{!LzQ1DUS; zUE{a8>4-xVXc7$9e%r_R#vd?l1x8K0PRqBcrh@jlV{K0oUuF4-o4m1CiZ^FUex&V5 zMAb^RO8z7m;x%cgVTcWCDiKw>*1?J^N=mplNse7q|Ej@fy_7!*QJiYn8Vxs(~6In>MRa@zQwQH zdsH^7tX7$;n)UgFWVqNHovDw zl#rpL5@hz){h{!X6V^=ptE6FaCcVk5qPa7)bOTQR*nj(n(1w4$#Q&i9C<5?@4?VyD z1c&^q+}{M@4vn{mb|E)WZ!YSWS$QM=QBGlB;2Hn|U;p{Vaep^aH#o6=)VUEJ@aOEf zy7Nz0?a{}|@}sKS;~U3%l;~j#-32w2!h%J5FtRKV48X>a1sK`D*c=9YJaGt&c#xbY z5D3_VJgShoC`BS$1_v-J%Yq7EbOjHx6q=0Mza=ST!CzfzZUA3G4>~kAw1|bF8ViKS z;Zvd_Qvi@9n!Fk?XIG4BWs!*;*~hHPlIqr5(B6A#i9^@;E<3f`WR9#&eWQ?}g`$?2 zan)Pol5#cDBggwoe{wPR)DIeO_E!@0`Z{CN?5AUOZ>?SOYo}B5q|1+JcJzgFQ>$vR> zwyEtRJ~H7%HkoN$)r@BX9sVpDqAo`LScIYB7$~T4G*l8)LY{D9LR5Gk9u^48f{-DQ z3QHP;l9CvN1_Xi_H!i;mUqLmW+>t7LkPdt_fEGl!ozGx_LyTrd*FVN;wPQjkNtrxA zEZ&Jl^+f$%U94z5($3B%_wYEtEcqbnL?hViHq<+aRIVxfJ@L^~Z!H>H6>XtB$Z*Tr zkmuL|-#7HVVRv0ss8E%5Bbj|FQ)mJz7M7wiX;heZ;_uOqJmk@mYV7oQ=!Ne`=;2LW zddcsd(Ua2Mp~a}E?E2Ch3%i=riX-;xwO6bK@bndQ{>6GcRz`b1%CGAnDMl8Gq& zWF65|#Qc(PKkJ}B%ZfvAVZhrg$o{s$}wpNc=qz4Gh!_S=5GPcA629T zb{6P60w1PEnT;1oKNlXyoLT;*axcdrU-OruZ+?)3h9*8;@e*R_p=;WJ0RXdN5&odz z2jek^tIB|okqAnn0%TGs4Dle}7nTtY%_l{LA_i<=2tHsDzDSRtLRDiT%`X)%UJp+y zRx6Mn6Tnl=qN0QN%BV5}aIhs9j~*W%O%2vh1>-}es4|BKLsO*LzA%I?ys4B>qf_eE znZOu|)gZmXU?>0(Fw6>sZ-{(cG(1Zz{?Ceq|;f#lRARo1HhthnE@WXa&kprm10I3150KBFDNz^=oQ3b#$yC{ z2JYDm7^S32N{jW#qS>F2D}f;pc6P2(GQvqY^7+Y3V->|^=Vm###)^t6Yh6bQq8Ku} z##8OnXe5`5N?LW6dyNOaT+K+W?I@|16cOo2pB%SkyltB*Kbtd5b!tSbBZVKYzQu*X zQLiL21O6NmoIyxgkO)TiK=Q{4;m}FioH(^NKn^~C`JzyiP*|)&m*RO8w~{PcGBY=uRQyzu*GEfBhyB~}%6hEQ#7*xFiOg-jANJ3+CdXVL`nZk@_Ei7#efe{}c0u)RDAH1Yy_Aa75U5ig3=tl;Aw**@XRdA7;H6(7R-n_Y@ZzSF z3zK`E+h*(@@9mk4(GVX?i>*bPvV`e6urokHYNdiA%`T_J1d7>aTp%Pc`>c}G&=GYw zBdS@D@Ax{)Qm%8nqCS|unv{@OHioj^MqY6VgZ*1O!ADEqsIAcF(di8b8b;FBMJt*q zmM`X|q__*%`WY5TxYV9bnWP5-7NZe032*0wRfi%&Se~)mP?(Fnb#@;9#59|9)!P|< zgh%~D;ksc{QB8JP%InL(^9piM7(=bhG~6*s zPLaqS;aYK?&eVJGRwNaVgiRv6-VaNK3mZJaUW2aBq}@mIgog}0BgV1-fAI;mNJ4#k z4?1dq5M5awx5QdDZy@ka3awqDUb7^Wlc(g9l;WOljHW@wlW54YN8V>a089_pjG?dY zyIIPkhABjD`J5~-xOU>$EIMq`J*MFI($L{tf5)AoI*)AKT$!j9>zw2ysaNGnTx-MU z$Z{F@(&8NZc0=Y^z6rZxuRjnaS$dE)kA)=W33pj!ydKx9gV$0`Zi(EOS;>I|On6l1 zrGpmz9N=^KBaVDX1Q2**g%-#nc(D{?z$$Co_k}%5r!`VY>El|tW9uL-!7;Bx=jKsX zU9+x(q#wJvTAeaOHCR9Jya>npk3p#Xt0Qd^X>5!bQ$k}#y#XNA@0ckvO1!(d3uRQ& zp;akfE>^wh*^HZ;QA3k6b2eA6?os}Ty!sQyRif{HX*-L@b_FX9lcXN#kw){e7M@{ai>ozFR-W^3_Rke*XYyn; z0H>DSdoJP{okkkCv%jiZjB3b;%BP8UqSH#7A3ETl?5=7@wt0+tWYI1aKVP^o!IO0O zpYq=~uh~ew&$6mI+9_RI0mbK9|eFqlW^+n9{l!f24`=oYF_I3>~?5-teV-(%xuL2}1L&Jl`07Rrg3@rNuW#_UGA|I=|e< zKr|6PX?G|#4~o(UjYl4_NAF+Mf2J2-k}G`Hs;L&HZH01{1U-XB9zjGsWF3LSabnLt zgtYjQY~C$?A`QJ{0l_KOS1ah%j#xdd1sy2Q5F)Z5`)Y@#T}fm|IPm3;u%bxrK(3xL zySAa8jg&h>o%&1{303ULE)v3@v`$vDy*j%7ozso|i_pz=+YErlgtDKAJ^;FElg`a1 zsJUHCA1eV%L1#@NA50aY8U6}~#(^s1Jf_wn75Jo%dsw*s?|GY_(--~u^7Y{k66jKOErc0lvszcAmY}X7>tt#QyDyij}qSC zQN;y?^%nqy1vl$boSfrx9sLIfhaseKTca&l^0$;Iz)--RiQtmkbv+!eOrIe)WT8I2 zBeBzg>I-eQh{quAw@T&W`%Jz)NlRJoY zY~SlOc)9Q{{P(G-pP!6vs!Dg)R#8@)M(lX=m&owqcGbxQ_9?^j46*3+DV2C6zD|xP zl=pn!h|NULbmn^|W7yVi*0f`-Bd~H#n|-sE?CwFs>1t7O>el$sWKY^ef@jTdE#4 zWo{^Z@u^rTts!(-$q-dblU@Z8CD%%XwUo`vR+$wek3V4?g*rd~qUeTMQ3VBcl`BS& zHbRU_alool*{L%oJ<^oJRv@#GZ!CKOzq~qWKB*4k-1NC@I9so%)AUR-C)fa|<@t?T zRB}9KWXVr)okR98cq9cOc~rY1#Y_Y6fs)!!<<&y$FMo;%2^73dBYdj8HTu)@@Tla% z+;CYYZ9#SOxNM7hO!=6xjl{RnoG7{&oS`wDG%1|E9juSdH+@*%$J!+QEcb}n5|w<$ zj%}8Qg%&BystM9j1Z9A5tGKMYOpUhe?$?iCIStkjj9Ct$dtRlchQ2qI$s%c_flM|w z=@6$!*xp+2_^y$7te9e1PM8Rkb9q zCABe5HOffojV)WYeNJvxIsRMtlc=Fh@j6J-c@Ja91lI7ub2e$i)SD%5a;XZL)mxry z8Jd&Hj;Ux)xQPbUj!j;1PSlahncp5~GS6pi3RTu06MyB^{qTeKJk1|nNYv&N?^mp{);hg9p<4t+p+DKA0;bziWqudr*O z;0%H6l!;Xi)FV!uxyR9({E3(PILf^fSJ3mZnwxPVIp+eM=GY%g57H{{*+M=p4l}QH zK;7S9Lf*P4qTbL4KO>b@Dq!3in7)WLeVO;-u-h08J|7wi$*DtWSe$B7BlO-f|%V^tk6BnKs7 z4+y`(acM*9ks01{;^c6VF3f8sd5e>Pt{3=szGk@6@G}Q|eXWOT=A)jwP0+W7s-C+E zYsL|%!jLI@#EsPTq-NdeoQN6}_G`?BLP<*o=dT~>6Y{&Gs*QG4ZRncDHbnAsvZ1kQ zRhR{|g&(q|El}WQJpPhJ^}gqIg4?0JBRrXYEYrs*^)47GkTXP%ni!eHPS{G)hxux^ zT_8Xzb__Ku+5lY%&VIg%KS!Z-V(j+PG0Ib^q-ep`?z($n2KmI1CVkk3Do47CJxwFA z>ql~KQbW;~msKIJ_GSwfl3j%&GhSrWx^|-toByGF+Mjydex^Jx9+~1b`TAgS!i~?I zq7wR<7fj={Xu_AC#1tH9XrH>#{7ZD1OEVMqt5ZpyS5|jO?JDN3hXT$|>&IIA8>|Ab zfN&v93p?KZmrxNIx0wO1QG=9p0>gI$FV)bA4(SYpebjYVVq>&Ncs!^-jj?r}k~R<6 zv8DBvV`42e*oAkGRmsa8EG4_QoK>arNJW=}Kds-U1aQKd!O1Ey74N&tta)MNFLFZY zUTs51Jq6gchug6VY#nTrhyCKr)6tIwbywmXYG~NggMTY744Lp}8BSSUk5IWbuC&jr zjg1^|j&woCrg*qHr?#?lO^u^}qDL^Lc!AiqJ5 zO$;QEk&2fGhpL#UKu1PHJS>jTq@N_08Hf&rE75RfYezqRPk{#MHG5XH%+8e z0lc5b%@fsAhB}|Jc)WJ`RlASby_chHk+Q0+-1Ww#pp*rn{J3JiK`t>UQOM0jkXZ4@^Bftp#SV9`$4lHB=`O>ll zc;p-zlA_)Be3mgU*PFT=dSBzeqKIlsl1%f!2?6f>_loAH_LMgY?c>Q^7vr)yyB0s7 zJ?naXHppm-W1QB-b;YJ+(-#EbMjmqU-E4QFfvu$riePLEMZ3KC)XWFr;fY1R%^ zvOy5ie>X4cSMmD5tSS%2x86bn$XjA;T=}pU|7hM_T}mwga9IunhyfH3@bu3IdU_xU zFGUBLdIbYeS_sQimSS^s#u(|x!+#lCA-%I*I?_kQv;@Ca_0LthRS1l)bUWdcw8r-&OgUN)JlAw;Be2+)Q`@BM4;_^KXo1 zrK#L@nr{#sEPP8s23iXE*KL4-3Z9}~(?%spUZ}?T>M2g4;yXt2Pe|JgdI5J zeH7rOOZ#r}jz#wNjk;N%*UUo_Q2&TyY8|g||F50ADDu|WSoP|!{TDU2UE6>CJj~1R zF!?HCm4X(#uY@?73PG;C-YGkfJB|sjLcKhRMYa(y_1>?ZVsB=0{7tR-)yS84^%STK zDsz4R=zp30%;V)Aj`>_wIUF`{?2X!dunu5_?=_6`goxjju$vIq{;lJmkk76lN5RVt z7pbIhwbF{J^ErF6=HTI;r{H*SULU*e@z1<}ZP8@-V;L0+^7W*PT3x`JFXaqxcPclw zXy$ULP2D9S=6~n^e;d}b2Hu5_PZxh~B0^3=il`OkbipVO50QCr3A5tixGdB5_-Tn2 znc0oxL>Kh_r+1X1(eu`r(qnD3MrUn)IQ<6F+Q{r)?i?R=`@?lWiQp+7|3>g|plua^!TM}P35TgSv=PO8m#zO`>nJB~ME=GE6< zyuG{KBqU;fM!<1kuzfEx=yYx4n#D*lUvv`Y@*VH-bZezeCkd)>KmIR*nm^jV=O+_7 z>63QQ#>TAArTrDU`>MPy!{@uU;&ic@!TWJvS->o>;xfKIYUaGA_}_Ppt#AZ1KS%`J zaIX0c=v!j6YRZQyeeNq}^C3uedr@@umPYk;i2Y>)fy_-p- zmiJB~-`2sS6F52ajt=X8B|U!i=yr9?__0y6p^wYL>f!}j$Ex%WTf9+C zzmJELX}FMpy!UTuma@Bqs#>CI7|o@oi+@8M$3*J;O#zUPk$TGHKq9NUP~6^z zh^_DE#pAGhs~}SYJO>JrB|0hd$gA ziSs%5qzdVqJQ;g;yNgFd5%G!yUaD)j`$RsLUoHgx46&fXIV%&L>Q^Pa*SP3~zCfJ5 z%CR4^SvhjcCN;2%nA(#0E0kkLL$KCL@hC&V_=DK~9&m2e?`0v961xTl-N)P1Lz!;G73Y^^APxrQ}S{7mo` zJ6Yk7{FP`c2^yD83E9fbCpp>1yA$STYqa$(Zn^K^{^8GO%$81yp|r4S8!aN$aP!Y2 zWxqDE+ergXCt6mdtZo^r{#38e({fCGultrcN-+CmuHt?R9epLj@cAFG5kK z|F9a`{J@D=g8w9O0V$uxI$X$?Y@>3c`Wq1S9DB$`j3eLN^(%a`|9Ed?CMh`h`%YVV z2B$UOU-Jq|-PB8)jq}>~8J+YZwYZlVUiE>%nlEj1=2^&V4O;hnkHFez>J^nvaF@KQ& literal 0 HcmV?d00001 From be6cda9a8fae99da524dda8e79764c177ad0eaa2 Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 17 Nov 2017 10:56:56 -0700 Subject: [PATCH 20/23] refactored process_replays to have load_replays method for single replay loading --- pysc2/bin/process_replays.py | 59 ++++++++++++++------------- pysc2/replay_parsers/action_parser.py | 1 + 2 files changed, 32 insertions(+), 28 deletions(-) diff --git a/pysc2/bin/process_replays.py b/pysc2/bin/process_replays.py index b6f38ac2d..f3d25e54c 100644 --- a/pysc2/bin/process_replays.py +++ b/pysc2/bin/process_replays.py @@ -122,34 +122,7 @@ def run(self): self._print("Empty queue, returning") return try: - replay_name = os.path.basename(replay_path) - self.stats.replay = replay_name - self._print("Got replay: %s" % replay_path) - self._update_stage("open replay file") - replay_data = self.run_config.replay_data(replay_path) - self._update_stage("replay_info") - info = controller.replay_info(replay_data) - self._print((" Replay Info %s " % replay_name).center(60, "-")) - self._print(info) - self._print("-" * 60) - if self.stats.parser.valid_replay(info, ping): - self.stats.parser.maps[info.map_name] += 1 - for player_info in info.player_info: - race_name = sc_common.Race.Name( - player_info.player_info.race_actual) - self.stats.parser.races[race_name] += 1 - map_data = None - if info.local_map_path: - self._update_stage("open map file") - map_data = self.run_config.map_data(info.local_map_path) - for player_id in [1, 2]: - self._print("Starting %s from player %s's perspective" % ( - replay_name, player_id)) - self.process_replay(controller, replay_data, map_data, - player_id, info, replay_name) - else: - self._print("Replay is invalid.") - self.stats.parser.invalid_replays.add(replay_name) + self.load_replay(replay_path, controller, ping) finally: self.replay_queue.task_done() self._update_stage("shutdown") @@ -159,6 +132,36 @@ def run(self): except KeyboardInterrupt: return + def load_replay(self, replay_path, controller, ping): + replay_name = os.path.basename(replay_path) + self.stats.replay = replay_name + self._print("Got replay: %s" % replay_path) + self._update_stage("open replay file") + replay_data = self.run_config.replay_data(replay_path) + self._update_stage("replay_info") + info = controller.replay_info(replay_data) + self._print((" Replay Info %s " % replay_name).center(60, "-")) + self._print(info) + self._print("-" * 60) + if self.stats.parser.valid_replay(info, ping): + self.stats.parser.maps[info.map_name] += 1 + for player_info in info.player_info: + race_name = sc_common.Race.Name( + player_info.player_info.race_actual) + self.stats.parser.races[race_name] += 1 + map_data = None + if info.local_map_path: + self._update_stage("open map file") + map_data = self.run_config.map_data(info.local_map_path) + for player_id in [1, 2]: + self._print("Starting %s from player %s's perspective" % ( + replay_name, player_id)) + self.process_replay(controller, replay_data, map_data, + player_id, info, replay_name) + else: + self._print("Replay is invalid.") + self.stats.parser.invalid_replays.add(replay_name) + def _print(self, s): for line in str(s).strip().splitlines(): print("[%s] %s" % (self.stats.proc_id, line)) diff --git a/pysc2/replay_parsers/action_parser.py b/pysc2/replay_parsers/action_parser.py index 4c0e869f9..a89cfb93b 100644 --- a/pysc2/replay_parsers/action_parser.py +++ b/pysc2/replay_parsers/action_parser.py @@ -54,6 +54,7 @@ def merge_dict(a, b): merge_dict(self.made_actions, other.made_actions) def valid_replay(self, info, ping): + return True """Make sure the replay isn't corrupt, and is worth looking at.""" if (info.HasField("error") or info.base_build != ping.base_build or # different game version From c72f8a9adeaa24c337868f23ecf4891ea0617236 Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 17 Nov 2017 11:19:14 -0700 Subject: [PATCH 21/23] removed required flag for replays argument, default is string 'None' and caught in path doesn't exist exception --- pysc2/bin/process_replays.py | 5 ++--- pysc2/tests/replay_parser_test.py | 25 +++++++++++-------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/pysc2/bin/process_replays.py b/pysc2/bin/process_replays.py index f3d25e54c..ba469a0f1 100644 --- a/pysc2/bin/process_replays.py +++ b/pysc2/bin/process_replays.py @@ -49,7 +49,7 @@ FLAGS = flags.FLAGS flags.DEFINE_integer("parallel", 1, "How many instances to run in parallel.") flags.DEFINE_integer("step_mul", 8, "How many game steps per observation.") -flags.DEFINE_string("replays", None, "Path to a directory of replays.") +flags.DEFINE_string("replays", "None", "Path to a directory of replays.") flags.DEFINE_string("parser", "pysc2.replay_parsers.action_parser.ActionParser", "Which parser to use in scrapping replay data") flags.DEFINE_string("data_dir", None, @@ -58,7 +58,6 @@ "Resolution for screen feature layers.") flags.DEFINE_integer("minimap_resolution", 16, "Resolution for minimap feature layers.") -flags.mark_flag_as_required("replays") FLAGS(sys.argv) interface = sc_pb.InterfaceOptions() @@ -260,7 +259,7 @@ def main(unused_argv): parser_cls = getattr(importlib.import_module(parser_module), parser_name) if not gfile.Exists(FLAGS.replays): - sys.exit("{} doesn't exist.".format(FLAGS.replays)) + sys.exit("Replay Path {} doesn't exist.".format(FLAGS.replays)) stats_queue = multiprocessing.Queue() stats_thread = threading.Thread(target=stats_printer, args=(stats_queue,parser_cls)) diff --git a/pysc2/tests/replay_parser_test.py b/pysc2/tests/replay_parser_test.py index 698b467f0..f2865bc97 100644 --- a/pysc2/tests/replay_parser_test.py +++ b/pysc2/tests/replay_parser_test.py @@ -12,14 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Process a test replay using the BaseParser replay parser""" +"""Process a test replay using the ActionParser replay parser""" from __future__ import absolute_import from __future__ import division from __future__ import print_function from pysc2 import run_configs -from pysc2.replay_parsers import base_parser +from pysc2.replay_parsers import action_parser +from pysc2.bin import process_replays from pysc2.tests import utils from absl.testing import absltest as basetest @@ -27,20 +28,16 @@ class TestBaseParser(utils.TestCase): - def boolean_valid_replay(self): + def __init__(self): + run_config = run_configs.get() + self.processor = process_replays.ReplayProcessor(proc_id = 0, + run_config = run_config, + replay_queue = None, + stats_queue = None, + parser_cls = action_parser) + #def boolean_valid_replay(self): - def test_random_agent(self): - steps = 100 - step_mul = 50 - with sc2_env.SC2Env( - map_name="Simple64", - step_mul=step_mul, - game_steps_per_episode=steps * step_mul) as env: - agent = random_agent.RandomAgent() - run_loop.run_loop([agent], env, steps) - - self.assertEqual(agent.steps, steps) if __name__ == "__main__": From 6aa04aa875dda28911b30b8ed28a740f7b3736e3 Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 17 Nov 2017 13:25:47 -0700 Subject: [PATCH 22/23] added unit tests for replay parsing --- pysc2/replay_parsers/action_parser.py | 5 +-- pysc2/tests/replay_parser_test.py | 51 +++++++++++++++++++++----- pysc2/tests/test_replay.SC2Replay | Bin 17436 -> 0 bytes 3 files changed, 44 insertions(+), 12 deletions(-) delete mode 100644 pysc2/tests/test_replay.SC2Replay diff --git a/pysc2/replay_parsers/action_parser.py b/pysc2/replay_parsers/action_parser.py index a89cfb93b..d087efc41 100644 --- a/pysc2/replay_parsers/action_parser.py +++ b/pysc2/replay_parsers/action_parser.py @@ -54,16 +54,15 @@ def merge_dict(a, b): merge_dict(self.made_actions, other.made_actions) def valid_replay(self, info, ping): - return True """Make sure the replay isn't corrupt, and is worth looking at.""" if (info.HasField("error") or info.base_build != ping.base_build or # different game version - info.game_duration_loops < 1000 or + info.game_duration_loops < 1 or len(info.player_info) != 2): # Probably corrupt, or just not interesting. return False for p in info.player_info: - if p.player_apm < 10 or p.player_mmr < 1000: + if p.player_apm < 10 or p.player_mmr < 0: # Low APM = player just standing around. # Low MMR = corrupt replay or player who is weak. return False diff --git a/pysc2/tests/replay_parser_test.py b/pysc2/tests/replay_parser_test.py index f2865bc97..144466c40 100644 --- a/pysc2/tests/replay_parser_test.py +++ b/pysc2/tests/replay_parser_test.py @@ -12,13 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Process a test replay using the ActionParser replay parser""" +"""Process a test replay using the replay parsers + A replay named "test_replay.SC2Replay" is required to exist in + the StarCraft II install Replay directory.""" from __future__ import absolute_import from __future__ import division from __future__ import print_function +import multiprocessing + from pysc2 import run_configs +from pysc2.replay_parsers import base_parser from pysc2.replay_parsers import action_parser from pysc2.bin import process_replays from pysc2.tests import utils @@ -27,18 +32,46 @@ class TestBaseParser(utils.TestCase): + + def test_true_valid_replay(self): + '''BaseParser returns valid_replay = True for all replays, + test the replay info loading and assert BaseParser does + return True for valid_replay call''' - def __init__(self): run_config = run_configs.get() - self.processor = process_replays.ReplayProcessor(proc_id = 0, - run_config = run_config, - replay_queue = None, - stats_queue = None, - parser_cls = action_parser) - - #def boolean_valid_replay(self): + processor = process_replays.ReplayProcessor(proc_id = 0, + run_config = run_config, + replay_queue = None, + stats_queue = None, + parser_cls = base_parser.BaseParser) + with run_config.start() as controller: + ping = controller.ping() + replay_path = "test_replay.SC2Replay" + replay_data = run_config.replay_data(replay_path) + info = controller.replay_info(replay_data) + self.assertTrue(processor.stats.parser.valid_replay(info, ping)) + def test_parse_replay(self): + '''Run the process_replay script for the test replay file and ensure + consistency of processing meta data''' + run_config = run_configs.get() + stats_queue = multiprocessing.Queue() + processor = process_replays.ReplayProcessor(proc_id = 0, + run_config = run_config, + replay_queue = None, + stats_queue = stats_queue, + parser_cls = action_parser.ActionParser) + with run_config.start() as controller: + ping = controller.ping() + replay_path = "test_replay.SC2Replay" + processor.load_replay(replay_path, controller, ping) + # Test replay count == 2 (one for each player persepctive in test replay) + self.assertEqual(processor.stats.parser.replays, 2) + # Ensure test replay is valid for ActionParser + self.assertFalse(processor.stats.parser.invalid_replays) + # Test parser processes more than 1 step from test replay + self.assertTrue(processor.stats.parser.steps > 0) if __name__ == "__main__": basetest.main() diff --git a/pysc2/tests/test_replay.SC2Replay b/pysc2/tests/test_replay.SC2Replay deleted file mode 100644 index ddc6d0c74ae2cb1b1b6415f50ff8a5100ac7c230..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17436 zcmeIZRa9KfzCF5t#wEBk5@;HiAdO2JcbDKU-8h5<4Z+>rgKKby;O-7V0|b&_fdmhc zKRJ7!JHB(qy$|>0zT7=~RM)R;RgYO!T~%vVscC9a1E>H1fB@k6gNIHGpmOSXT6)NM zSlW5gD=E=?*t$7c`cU)oqN3vg*r=%37$9tB3~W>^?70+79PGYCOb|8*1H1kW3Kj+m z4mv6-D*ajuLV-1KN`Y#=HM8$MG8FG`R)YUZdZ^-mfDq$<1pgEG4+H;U;QvDg)HJog za}V{=zySaaKnG|f9_*(8K>rX&_~8DDzj^**U+iCg*uVY%`v3SFh4QcX|Nhqax54_~ z@o)f8D5P5!gIC3#Ry?Eg^W*C#lBG=?RfXP)&ZSnV$9+Bi@3Q~x-wIx z8zDq^r3{&M41F+7U;rTS`sUA-o}Qjo8VCzR4UjC~-hrYJ1_B^dI1V5UN0H(I?zv=? zCDh)~;?7KDnK7wB{okTA^3JAw;K^H} zG0bRoy5BKCyVYXoq%Lc=IGTZ=1D<#HvQSVbBvhJQMbXHf^OQzG(>A${*N!geGFFS7 zM4mjf{i@v2-Y;OKRK})CsWRN#4X%7&Ld&YzB(5mAohrT?cx}T_>b~h8OhIYGP9J>6 zU`IW0Lxzfl+bUy2>dqJa7)Lao|G5ws18r3JS%2C;}oXWzA zE>#uDOX@z9%9^Hf+m*~V5E?Qjx?9j@fm-o6LI@}0-uIeQ9me_(mhnHCDgc1s|72P4 z|AyrsNNi2?M>?ce)H)8%GV)h&z*jyfM;E}#f)&^m{G}M>2*8-Y*O{-vuPorjivs|R zI6fGFs!89-ie?C}qNfQA(E`i%;bXufS(RYP@PvBXp(AI}(|bj2se$DM4eg5G3X|A% zHXA!exZ~kWQOuTI65eYE#rzDY^{=heOmT}@5&KW^g1<~ZEt+YUUy zT^Y6}3G0<$F+;Ho902GFmL=r#ODrj6PBqdrJ(%FBux#uHR2AFNZotYsQ`KGje2?dB$jb9hSL6C!JcS{oXbgFX)glo>1`sXD zolu@01MDyuOI1~rh)B1;bSY^Xqa(~qICnX{B%zC~z0&(JhS@{{Z3#xb<-*G9*wROh zG}P3PdClHx^a5sT3`I)}P(!~=ooAlmINpn0jtO~%lAn4|#gt?&oc5D1?$#vIblR}Q z;%^LLs&dl#k~q@q>Kuwg+>}ZcBsk;IKLlv^>HfrYKyq0X2?mE&0{^_pGL{%1qic@|3 zyic`Jd^4H!-$(Pvm+*LRScf8G-3Tf9-R9ClHG|ue1SGWhpxOX0okCI~OK%{ypHa-T z)K;0^6rhTkLCU_Ob&~XM11S|ERQ#FRVr_3Ze!wlTH!Sy2cYUdXwwkSDig-z|FB>xH z4F?C}r+dqbkNPaQKN{OKLksx%dGJGMXT)Yr@3^hA2as>C+Fm-oc=qy+0k_bHgH41Y zGNuO!5ZVBnvhrj%+yB)s89cs=N>}`@*X-rup3)R|?o0UrkNpSJ#SA+S>%!L$O#c`L zCH`d^hg0@rnH=i>VtSw+SZgBE(BcCl7UB2+O2HfegW+T3XeG``sY#Vr7R*up9oixA zzeEq~EK8^^Q1L4j%Pb%kiIR?N=99|isW6y{3iB($j5J=<3uf>PT51F_mu~k#*V>63xNKT`zjiws<{{8dpZ0 zUVVP)y}f(ii2Y}iA4tKoxTXi|v)3=H4{ za}G!t8X_+a3{dH@Wi0Qj#$0P;r_nTBfd5=voN zY={i}Z{h zhdWl(H1yFRLS1OGZYz@slTlTDuu^=o#iBER*~2WYL~pHD$T$4hcOkKg{lGw)wJ3>4 z1yoQ_Fg8(e#Ka>HR;5O;2fc1x04X_?MzSfUvl~RpX#rvSqP{1Y&F_4S2hu%S70Pn< z#{YDdetmtSPxR_$usvRjSD26By7ZZ3p!BQl?;N73CgUe{dDtZlVU$MIB^Aid&b5-a z>+J0eHb$93=moh*sVt%3MPg=ud%F-R$K7u=GQ%-Ej}KpT)O3$j-H6tnmIm!25#4XqHUcDD{RyZA;f_s9eW%+=!zq8$j-e%cgD-@zGS5c3>PQQWEL=VcD6RouUM z#qFV0MWN$?5NczeKGYz`-t{Pcl_R_B*?uZ7{o?kV3oo6Yxq*`+A~ah@?c#(}-gbZ~ z$#l#PG}!kIA-9|KZ3NMGS|Awf$OhAlE*~Ly z+n`ICr4B2|MIh_&)(LS;q6M?GfGkn@+H|8v(9k7wt?j ztDGJsI|1@$mNVC_}NBTO3D|P^Zisp4TM#vNPj^ zYOVJ#qJ1tn=Td*}D^uedILt)8SV+^I91?zCTHjNDZ!h-)yp@hOq;^rLAwtCJ5-+oA zp6{5k<}hs4QcAB}MA4ui8&tfBrtyKHxn|Jmh{i}t`aRYO2CTZ|(RT|%@>Nf2kh(IB z*{}MEAozLE#AUWqVuJyH=)ozym8MDtwre>?hi0pt-A!Uwu!@v^sRK0qtHAoATpm}| zfjCBr2!e2bEHx{$HGQst45f#H#(W5EUG_K7$I>;uj=`~?hxFDtRDd?q2hm4r7m-Tg zPO|su2OCA7sfxh~`&s*wI94fX>!rydlpEF#H7`RLiomkW)jBw89C*}ZZAu8`KHd5n z&SJNMjYj9BM_XL`6aB=#sxf#@hP>AO24lz!Zvz(-E9Yu3yZ9IfgVD*lii?c(QVf%w z9V=JAAv`VJZX56CS-R`}me=X0BfqZiof*yP^rGv=yfvQ%ubx!~7SZP;6}4?d_w>Q2 z8`50DPD4qc$ZSx_Twz>@wM_)GEwMIpCShIJdIJ+SYGHvh;OtP3Y^ipe8ySbv48W7E ziHX&KAqHOh3Vj2Wrb;p>%*Zn^nBD@_zy&O&`=h@3e%fmb|F`b=O|P1H$imu}z7dn4 z5J~6H=kXqeWXs5nw%K=9dEw3HptgnU7X?DgI6*|2(eV}}@f=pD~$;e)WW_`*qADb=1Y;jP!=-F8etn>iJjmN5H%s;ZsSyU5cc(G;#{4 zJ!Ifh8?g~0AKeBQtXy4XA_q#1dK$2rzT?`$$<*@1k$FhEAT^mCBwbLU3yDS{(aIF( zKHRwULQu=2nk)IUCWb~sbs>;YLN*jEHlPBPK9LlgI2s}=8yqRkiUpTtBO_A8DU@SX zl!dTI5=keKMkP3lmzl?^7`S#`tGvev1xjS4=p&;POL3xAk_}3*IBJR&hN;ShsBqW(1LG@vwp{3PMTt7C~h?#Cj9k=YX z{bVNE7l>~%IlpK>GndTo)k`w>om#rF8;UP2@d!r$*7*T#eFZJ;>UokgRxNLzR%}Vn zla@S2^L=g$ZmBb03aZdCI~J*_+B3Q$NZGq))uiV5k3Yg@Tm0u(x4_OU8%Fy}P{%|# zR)Gcns0Osx8ivP$BE18>cgjMnQ&SDd=IhRljNo@jyMsIyQj17obU7nI4%x-8X!ov9 zF_P3QZ5+5K2#;yeH|V%qD^bJvRM0%DW*PjxZTI7@qH8x$#t z)hUw^77THMXx3JQ%Q#X@Ec9B-Ohmdt(9EjTBH_K{JXmo_ZQ@XWDKO{7<&pdHt3u0O zA%5<*X$(L2L6Bu08k%}|5(qXTIEE@bwMG+CT6APpF^iqzTp&`Gs4)(wApt?1Lz5g6 zHUsjjw|nlJ&wtRI-h_O8k^dxN*rjo6*yV`sl_-)%Jr*w!nlX#%xy#xwyGV5A=Y~B#O4ByO+ND_sX@#*>4Bl1FG>>F zsb5(T7h;z^7C26O!IzBhNr)X63!h_vkxFY~$Dzr97F87>V&ppnhSw(qNhN`I#*>*j z+oN77S*y<_YHpnWjE=oZtS+q?wtSH+2JWP3?RgHJ>HK}NllTIAv&x2`P&A6-aBv(9 z9|=Y&#UtuJP|o}Dfkl8OOS5q}aDf~D#Yt(g|3Fzf+5m^XOIFOi;JFHP+x1;vWiHy= z5Ir{fs~Q>sqOd%<6Q#LoXtt2*uD9bM#HS@usrhNe7k#@#YdJ5--Hd=;^Q#C=TTu#} z%+Oe|XAO)pys(Oma1i&`Wh8F3C90ntgm~5{rHkSOduT2z=ow~Ho=%TL=^-@8g1^=-XDzcftGYEeuV5@+vy~ccO`qCXiw}+>7#7z8kO+yBQQM$+B^rFk|+D}5cc>$05^nPS8kE*HR z_I;@bk8*+R+(KLyS@H}bUh+HJP{CUpso3vy`E2FL=h7MikXP(6Sznx6g_F@M7~O|J z(jhI>6(3!-f}`;kP@>j{HtH@Sdq0k$f>7m1v0cu^mej~%C2>5PiEzV__e-{Lct!wDrW|oF1j}Az3QYgF z-I$l?J08#Rx3xCc)+GdSGs#pbcM77ptLv7wJN>KpB9UYUtCVuLrDW+7@9Wg$wOP_~ zX_SP_(lS;pQ!Vo@W)Y?e`E>33%W2K|kq!E_kF=t5Dw`5j6MZhm`d=69$ZAPDe$dJV z6?mwG?*e9i`zcy$Jcu=gAHSgVMl>`PbCRg2mnZct+Vn|y3Q#_+_9*phJv-NzKv6s( zx4@Je!1>utb9_YZ3*61%`pvg5JkoqL|$oiDpbeD9jvW&(K>fh_f`+Mav-JyofEtx&e=-6sq^w*Duad*-C7&V?}KE zRoCjDsQa+HqL|nHtsCITMQIQzWHU)S`| z;kcrY9M1Fc)4gk?*(xZT&AP`X=_^Fl&nyn9(rs^pV=f3s(wrGjHyiB8A(OaeWM)&A z9prqJr7P?_+uD@;w1$4W1S=m8vy6PnRIBh3lg+TWue@;?^~Lg|*_nSWrm!-s;J*(# zI+8V$!TS*A%$vg>V$#J~vW_1~;hLGj)k2{7*+AW<4m`b4##M1V3c)zt4B;P}5DBIA z)hWvsoyH`6ZjerP=J@$Xwgbz6WS$-enLS)awB#Db#h7J+#aF}NPM_gcn3j0THVtFh zr-vxxFcCFPO$#`$z3;6GEtYeiMqR{wszp#Sp{f@j-Nn%lX4lt45Tzb_pimAQi7Zxt zokcwhYOjr3`Pb|b0#hVI~7!i)VI~ZtBMQ10W zW+MTetL!g`NzXn}e*Zm$*Jp$rL?nmpT)yVsg3zUY2DB^br_LN>RqcA<37ZcZq@E~8 za(G;qVsl9QKf@2uUklAYUVQ}G`*fe#eKhTnq_AtE9jR8>@XVs-#MAqw`_W|3%);jA zfz$E}|5|;FutoitbT2i+^DH9Rg4F9rzF1RpCq?`8;+Hdq$mm6S*c3J;#sTG zzD$ITZdiWnA8kg)YflC87DC*o%`#i`Q54C)nx9Ej$=VSXVDON-bMttTy~0N#1xdxY z?Ce&>0=p1r3_SrqIcdrIR0e0Zq;$oREjoVF=qoW$>>s3|5r!M9;4CXrB?*>Lkvpuav zvq)6HBf%~4(aYJZcsXhfvDDPaSZfRctM)dr)Dh;j1QfhV?UPHB9PO2uyxxY_G5yb< zC_WmGu3#pz@?`DPWMyYGAPhHT+==F!uiq9E8v?TU@m+M6E<%iJ}T2RUxM~-^2 zwMg)5tByt6U$}j^pG^zE7vUw|T2d;0k@YU&GAU;iT+MS||77eIn&UotYt9V}v;Ha?js4b|^e-M9uD}lTj$)28l zB20j4lVFC^P$jhKRwgo`6D{K<;aJ=(yQyG7Oa*y~#a!89F|wJmdHv4*42oB#t1Elm zI8!=j2dPw$tjY21hc z^C*&UUcr5BOv()@m}8=Xj$POg>kMIFV8tP7=C^N#ZjauTQXc$9_rE&na>+=6H%jqX zPz4TA6q(C~Z2QMI3_GOlx`tv(Pi3%%e^lV%l}jj#(fYX=whCU(Ux90E9Pv4OmPPlp zM&ZSnm@doZAIaObR4TO8J1L;fO4~LeM7XVu8+bGG>n+=nk4#2Hi&yK`9mWaTeKUwz zlexA>3_0AGER%TAKej6EX2GSt;rN!Bgf-%Ypl%7m-T5$gfXeWhx|Jt)ocMl63!Fk)n{0YWj z;ZluJx02tI_ouwp(Sud``0T7TilLb*HgtFk76~Qa2m-k!GE%lG(TsrP0lt_S?H*HW zv976>A1)Nt%vVG&p0W2*9Q(g{Gk)K(mrC)echhJRWc2ph;ZDd#pk1P)R+ozm&Ua8&6q^S`wG4z}SC6dvI0L)<;Qw8rZT zv)T-b3<_%SXf%BJS?FEfuN$Yo8xmxI1yXDoQ=#ygwFSRdI%Fwd@>2 z#uLi_xTmQmi8OFr(twBvq@yGCUO>5~?p^5Uka0z@%P zO!YD0-DY=kc)98N99(c`m?_)##GigQTYg6_sr2Hig#J*weYFLcy^OBV$*5(PIA9{4 zm(awPJ^Rt)2-*lZz_UmgwRgw=QgeJ5fIESEs$%PC#C%Xat?t?R&SVOW?#%So^aYdM zZjXt(MaWNGss27T0Dh1uDn`z(SS29{Kq!`t=Wfy|AuRu01_r zDizE}ItLj8gCh6<&)~e3SwrIa$H26jlI|Bmx61C4mE%cIMv)Qc2&&ZE>I<|NMkpb z*o4LhGbzF16);uce%D>%QJH?JHf5Yw@6Q65%zXznwjH2!O4WPbio>MH5 zZ3D|^cs1P+e*Sn{M0>hUaC{(_ftpujmMjst~c6?7cSE3aL{( zWWZ3SE7u(9KiBMpFodv&a%!lakJYOC@eyKa1SHq; z|Hi!fYH7eUF=va#_CI+FJg8U;WsfLgMn+LBxL4FsOOuvdK<7A zGjz_SH)+rjcR1CQkwvLQlg_1hPhewTuDSwxF9_<(cG`nh|D<1suzwDiI6hp*g=tKYI*5#inp7us?a8Px(rIXtD>41L?jBWj!9C6|$+*1_l@)zo zYyfJP2#p-mQZk#NSWJBLR3QjW2%oiWEAr_)wiZJ~LCyG^h!j&QeHoLePeeRl^M6N! zA+}LLCsyO4WzQV~Sl$-AdMVefqPC#^iRLjQX`k`mf- zTU&&Mg$;giF3mj;N-Hp*JhD$FqDdM=10_gO`mr4YV^}rq+x|B4o z3T0#{G(Gq((BaN7d!HYI9hnCh+ih8b#E3U0AkfEQ-qg`u`2kw1?8u_}1DHwV6Z(95 z*-VD!Sp4{7nYUXXtsTD!_(}Tln>wQP#zivxsdFUW=R0`m>4xDdUM%Y3f3b1m`m)-k zoBhI>>DAgsX^EV*>^5PK>2E_b^oQZ`ajnNF2R+nZdx8=^tEKEB^*j9+9$}lvd*k0d zX&l_nb=dL2Vf*CW`3oX-TCI|idEc0~*0n_{OmUK@PB}Y|KZmoN|9l_TRQc|ghWh}5 zxPs-699dZFZvUr6oxr=k&yX@9%WT)7Y0Us*C)6M79ETU$cfUaKS+gAG{!LzQ1DUS; zUE{a8>4-xVXc7$9e%r_R#vd?l1x8K0PRqBcrh@jlV{K0oUuF4-o4m1CiZ^FUex&V5 zMAb^RO8z7m;x%cgVTcWCDiKw>*1?J^N=mplNse7q|Ej@fy_7!*QJiYn8Vxs(~6In>MRa@zQwQH zdsH^7tX7$;n)UgFWVqNHovDw zl#rpL5@hz){h{!X6V^=ptE6FaCcVk5qPa7)bOTQR*nj(n(1w4$#Q&i9C<5?@4?VyD z1c&^q+}{M@4vn{mb|E)WZ!YSWS$QM=QBGlB;2Hn|U;p{Vaep^aH#o6=)VUEJ@aOEf zy7Nz0?a{}|@}sKS;~U3%l;~j#-32w2!h%J5FtRKV48X>a1sK`D*c=9YJaGt&c#xbY z5D3_VJgShoC`BS$1_v-J%Yq7EbOjHx6q=0Mza=ST!CzfzZUA3G4>~kAw1|bF8ViKS z;Zvd_Qvi@9n!Fk?XIG4BWs!*;*~hHPlIqr5(B6A#i9^@;E<3f`WR9#&eWQ?}g`$?2 zan)Pol5#cDBggwoe{wPR)DIeO_E!@0`Z{CN?5AUOZ>?SOYo}B5q|1+JcJzgFQ>$vR> zwyEtRJ~H7%HkoN$)r@BX9sVpDqAo`LScIYB7$~T4G*l8)LY{D9LR5Gk9u^48f{-DQ z3QHP;l9CvN1_Xi_H!i;mUqLmW+>t7LkPdt_fEGl!ozGx_LyTrd*FVN;wPQjkNtrxA zEZ&Jl^+f$%U94z5($3B%_wYEtEcqbnL?hViHq<+aRIVxfJ@L^~Z!H>H6>XtB$Z*Tr zkmuL|-#7HVVRv0ss8E%5Bbj|FQ)mJz7M7wiX;heZ;_uOqJmk@mYV7oQ=!Ne`=;2LW zddcsd(Ua2Mp~a}E?E2Ch3%i=riX-;xwO6bK@bndQ{>6GcRz`b1%CGAnDMl8Gq& zWF65|#Qc(PKkJ}B%ZfvAVZhrg$o{s$}wpNc=qz4Gh!_S=5GPcA629T zb{6P60w1PEnT;1oKNlXyoLT;*axcdrU-OruZ+?)3h9*8;@e*R_p=;WJ0RXdN5&odz z2jek^tIB|okqAnn0%TGs4Dle}7nTtY%_l{LA_i<=2tHsDzDSRtLRDiT%`X)%UJp+y zRx6Mn6Tnl=qN0QN%BV5}aIhs9j~*W%O%2vh1>-}es4|BKLsO*LzA%I?ys4B>qf_eE znZOu|)gZmXU?>0(Fw6>sZ-{(cG(1Zz{?Ceq|;f#lRARo1HhthnE@WXa&kprm10I3150KBFDNz^=oQ3b#$yC{ z2JYDm7^S32N{jW#qS>F2D}f;pc6P2(GQvqY^7+Y3V->|^=Vm###)^t6Yh6bQq8Ku} z##8OnXe5`5N?LW6dyNOaT+K+W?I@|16cOo2pB%SkyltB*Kbtd5b!tSbBZVKYzQu*X zQLiL21O6NmoIyxgkO)TiK=Q{4;m}FioH(^NKn^~C`JzyiP*|)&m*RO8w~{PcGBY=uRQyzu*GEfBhyB~}%6hEQ#7*xFiOg-jANJ3+CdXVL`nZk@_Ei7#efe{}c0u)RDAH1Yy_Aa75U5ig3=tl;Aw**@XRdA7;H6(7R-n_Y@ZzSF z3zK`E+h*(@@9mk4(GVX?i>*bPvV`e6urokHYNdiA%`T_J1d7>aTp%Pc`>c}G&=GYw zBdS@D@Ax{)Qm%8nqCS|unv{@OHioj^MqY6VgZ*1O!ADEqsIAcF(di8b8b;FBMJt*q zmM`X|q__*%`WY5TxYV9bnWP5-7NZe032*0wRfi%&Se~)mP?(Fnb#@;9#59|9)!P|< zgh%~D;ksc{QB8JP%InL(^9piM7(=bhG~6*s zPLaqS;aYK?&eVJGRwNaVgiRv6-VaNK3mZJaUW2aBq}@mIgog}0BgV1-fAI;mNJ4#k z4?1dq5M5awx5QdDZy@ka3awqDUb7^Wlc(g9l;WOljHW@wlW54YN8V>a089_pjG?dY zyIIPkhABjD`J5~-xOU>$EIMq`J*MFI($L{tf5)AoI*)AKT$!j9>zw2ysaNGnTx-MU z$Z{F@(&8NZc0=Y^z6rZxuRjnaS$dE)kA)=W33pj!ydKx9gV$0`Zi(EOS;>I|On6l1 zrGpmz9N=^KBaVDX1Q2**g%-#nc(D{?z$$Co_k}%5r!`VY>El|tW9uL-!7;Bx=jKsX zU9+x(q#wJvTAeaOHCR9Jya>npk3p#Xt0Qd^X>5!bQ$k}#y#XNA@0ckvO1!(d3uRQ& zp;akfE>^wh*^HZ;QA3k6b2eA6?os}Ty!sQyRif{HX*-L@b_FX9lcXN#kw){e7M@{ai>ozFR-W^3_Rke*XYyn; z0H>DSdoJP{okkkCv%jiZjB3b;%BP8UqSH#7A3ETl?5=7@wt0+tWYI1aKVP^o!IO0O zpYq=~uh~ew&$6mI+9_RI0mbK9|eFqlW^+n9{l!f24`=oYF_I3>~?5-teV-(%xuL2}1L&Jl`07Rrg3@rNuW#_UGA|I=|e< zKr|6PX?G|#4~o(UjYl4_NAF+Mf2J2-k}G`Hs;L&HZH01{1U-XB9zjGsWF3LSabnLt zgtYjQY~C$?A`QJ{0l_KOS1ah%j#xdd1sy2Q5F)Z5`)Y@#T}fm|IPm3;u%bxrK(3xL zySAa8jg&h>o%&1{303ULE)v3@v`$vDy*j%7ozso|i_pz=+YErlgtDKAJ^;FElg`a1 zsJUHCA1eV%L1#@NA50aY8U6}~#(^s1Jf_wn75Jo%dsw*s?|GY_(--~u^7Y{k66jKOErc0lvszcAmY}X7>tt#QyDyij}qSC zQN;y?^%nqy1vl$boSfrx9sLIfhaseKTca&l^0$;Iz)--RiQtmkbv+!eOrIe)WT8I2 zBeBzg>I-eQh{quAw@T&W`%Jz)NlRJoY zY~SlOc)9Q{{P(G-pP!6vs!Dg)R#8@)M(lX=m&owqcGbxQ_9?^j46*3+DV2C6zD|xP zl=pn!h|NULbmn^|W7yVi*0f`-Bd~H#n|-sE?CwFs>1t7O>el$sWKY^ef@jTdE#4 zWo{^Z@u^rTts!(-$q-dblU@Z8CD%%XwUo`vR+$wek3V4?g*rd~qUeTMQ3VBcl`BS& zHbRU_alool*{L%oJ<^oJRv@#GZ!CKOzq~qWKB*4k-1NC@I9so%)AUR-C)fa|<@t?T zRB}9KWXVr)okR98cq9cOc~rY1#Y_Y6fs)!!<<&y$FMo;%2^73dBYdj8HTu)@@Tla% z+;CYYZ9#SOxNM7hO!=6xjl{RnoG7{&oS`wDG%1|E9juSdH+@*%$J!+QEcb}n5|w<$ zj%}8Qg%&BystM9j1Z9A5tGKMYOpUhe?$?iCIStkjj9Ct$dtRlchQ2qI$s%c_flM|w z=@6$!*xp+2_^y$7te9e1PM8Rkb9q zCABe5HOffojV)WYeNJvxIsRMtlc=Fh@j6J-c@Ja91lI7ub2e$i)SD%5a;XZL)mxry z8Jd&Hj;Ux)xQPbUj!j;1PSlahncp5~GS6pi3RTu06MyB^{qTeKJk1|nNYv&N?^mp{);hg9p<4t+p+DKA0;bziWqudr*O z;0%H6l!;Xi)FV!uxyR9({E3(PILf^fSJ3mZnwxPVIp+eM=GY%g57H{{*+M=p4l}QH zK;7S9Lf*P4qTbL4KO>b@Dq!3in7)WLeVO;-u-h08J|7wi$*DtWSe$B7BlO-f|%V^tk6BnKs7 z4+y`(acM*9ks01{;^c6VF3f8sd5e>Pt{3=szGk@6@G}Q|eXWOT=A)jwP0+W7s-C+E zYsL|%!jLI@#EsPTq-NdeoQN6}_G`?BLP<*o=dT~>6Y{&Gs*QG4ZRncDHbnAsvZ1kQ zRhR{|g&(q|El}WQJpPhJ^}gqIg4?0JBRrXYEYrs*^)47GkTXP%ni!eHPS{G)hxux^ zT_8Xzb__Ku+5lY%&VIg%KS!Z-V(j+PG0Ib^q-ep`?z($n2KmI1CVkk3Do47CJxwFA z>ql~KQbW;~msKIJ_GSwfl3j%&GhSrWx^|-toByGF+Mjydex^Jx9+~1b`TAgS!i~?I zq7wR<7fj={Xu_AC#1tH9XrH>#{7ZD1OEVMqt5ZpyS5|jO?JDN3hXT$|>&IIA8>|Ab zfN&v93p?KZmrxNIx0wO1QG=9p0>gI$FV)bA4(SYpebjYVVq>&Ncs!^-jj?r}k~R<6 zv8DBvV`42e*oAkGRmsa8EG4_QoK>arNJW=}Kds-U1aQKd!O1Ey74N&tta)MNFLFZY zUTs51Jq6gchug6VY#nTrhyCKr)6tIwbywmXYG~NggMTY744Lp}8BSSUk5IWbuC&jr zjg1^|j&woCrg*qHr?#?lO^u^}qDL^Lc!AiqJ5 zO$;QEk&2fGhpL#UKu1PHJS>jTq@N_08Hf&rE75RfYezqRPk{#MHG5XH%+8e z0lc5b%@fsAhB}|Jc)WJ`RlASby_chHk+Q0+-1Ww#pp*rn{J3JiK`t>UQOM0jkXZ4@^Bftp#SV9`$4lHB=`O>ll zc;p-zlA_)Be3mgU*PFT=dSBzeqKIlsl1%f!2?6f>_loAH_LMgY?c>Q^7vr)yyB0s7 zJ?naXHppm-W1QB-b;YJ+(-#EbMjmqU-E4QFfvu$riePLEMZ3KC)XWFr;fY1R%^ zvOy5ie>X4cSMmD5tSS%2x86bn$XjA;T=}pU|7hM_T}mwga9IunhyfH3@bu3IdU_xU zFGUBLdIbYeS_sQimSS^s#u(|x!+#lCA-%I*I?_kQv;@Ca_0LthRS1l)bUWdcw8r-&OgUN)JlAw;Be2+)Q`@BM4;_^KXo1 zrK#L@nr{#sEPP8s23iXE*KL4-3Z9}~(?%spUZ}?T>M2g4;yXt2Pe|JgdI5J zeH7rOOZ#r}jz#wNjk;N%*UUo_Q2&TyY8|g||F50ADDu|WSoP|!{TDU2UE6>CJj~1R zF!?HCm4X(#uY@?73PG;C-YGkfJB|sjLcKhRMYa(y_1>?ZVsB=0{7tR-)yS84^%STK zDsz4R=zp30%;V)Aj`>_wIUF`{?2X!dunu5_?=_6`goxjju$vIq{;lJmkk76lN5RVt z7pbIhwbF{J^ErF6=HTI;r{H*SULU*e@z1<}ZP8@-V;L0+^7W*PT3x`JFXaqxcPclw zXy$ULP2D9S=6~n^e;d}b2Hu5_PZxh~B0^3=il`OkbipVO50QCr3A5tixGdB5_-Tn2 znc0oxL>Kh_r+1X1(eu`r(qnD3MrUn)IQ<6F+Q{r)?i?R=`@?lWiQp+7|3>g|plua^!TM}P35TgSv=PO8m#zO`>nJB~ME=GE6< zyuG{KBqU;fM!<1kuzfEx=yYx4n#D*lUvv`Y@*VH-bZezeCkd)>KmIR*nm^jV=O+_7 z>63QQ#>TAArTrDU`>MPy!{@uU;&ic@!TWJvS->o>;xfKIYUaGA_}_Ppt#AZ1KS%`J zaIX0c=v!j6YRZQyeeNq}^C3uedr@@umPYk;i2Y>)fy_-p- zmiJB~-`2sS6F52ajt=X8B|U!i=yr9?__0y6p^wYL>f!}j$Ex%WTf9+C zzmJELX}FMpy!UTuma@Bqs#>CI7|o@oi+@8M$3*J;O#zUPk$TGHKq9NUP~6^z zh^_DE#pAGhs~}SYJO>JrB|0hd$gA ziSs%5qzdVqJQ;g;yNgFd5%G!yUaD)j`$RsLUoHgx46&fXIV%&L>Q^Pa*SP3~zCfJ5 z%CR4^SvhjcCN;2%nA(#0E0kkLL$KCL@hC&V_=DK~9&m2e?`0v961xTl-N)P1Lz!;G73Y^^APxrQ}S{7mo` zJ6Yk7{FP`c2^yD83E9fbCpp>1yA$STYqa$(Zn^K^{^8GO%$81yp|r4S8!aN$aP!Y2 zWxqDE+ergXCt6mdtZo^r{#38e({fCGultrcN-+CmuHt?R9epLj@cAFG5kK z|F9a`{J@D=g8w9O0V$uxI$X$?Y@>3c`Wq1S9DB$`j3eLN^(%a`|9Ed?CMh`h`%YVV z2B$UOU-Jq|-PB8)jq}>~8J+YZwYZlVUiE>%nlEj1=2^&V4O;hnkHFez>J^nvaF@KQ& From 14d732d658758b0da24f5fe5432c5d221b7ed48e Mon Sep 17 00:00:00 2001 From: Cole MacLean Date: Fri, 17 Nov 2017 13:26:23 -0700 Subject: [PATCH 23/23] remove debug lines --- pysc2/bin/process_replays.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pysc2/bin/process_replays.py b/pysc2/bin/process_replays.py index ba469a0f1..2129b6484 100644 --- a/pysc2/bin/process_replays.py +++ b/pysc2/bin/process_replays.py @@ -58,7 +58,6 @@ "Resolution for screen feature layers.") flags.DEFINE_integer("minimap_resolution", 16, "Resolution for minimap feature layers.") -FLAGS(sys.argv) interface = sc_pb.InterfaceOptions() interface.raw = True