From 4d0eeecbba620ca47ea9f124a47315551c12e63f Mon Sep 17 00:00:00 2001 From: Jan Kwakkel Date: Mon, 22 Jan 2024 00:46:21 +0100 Subject: [PATCH] Minor edits to benchmarking code (#1985) * Make RandomActivationByType.agents_by_type backward compatible * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * updated warning * Update .gitignore * Update global_benchmark.py * pep8 rename * Update schelling.py * Update schelling.py * Update schelling.py * Update schelling.py * Squashed commit of the following: commit c0de4a1c4dbb586f26084764d6f34818ce1e5dfb Author: Ewout ter Hoeven Date: Sun Jan 21 14:43:08 2024 +0100 Add CI workflow for performance benchmarks (#1983) This commit introduces a new GitHub Actions workflow for performance benchmarking. The workflow is triggered on `pull_request_target` events and can be triggered with a "/rerun-benchmarks" comment in the PR. It should be compatible with PRs from both forks and the main repository. It includes steps for setting up the environment, checking out the PR and main branches, installing dependencies, and running benchmarks on both branches. The results are then compared, encoded, and posted as a comment on the PR. * Revert "Squashed commit of the following:" This reverts commit 948508754eaabb2312cef6863c467ea905069551. * Update .gitignore * further updates * Update benchmarks/WolfSheep/__init__.py --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .gitignore | 4 +- benchmarks/Flocking/Flocking.py | 80 ----------------- benchmarks/Flocking/{boid.py => flocking.py} | 90 ++++++++++++++++++- .../Schelling/{Schelling.py => schelling.py} | 30 ++++--- benchmarks/WolfSheep/__init__.py | 14 +++ .../WolfSheep/{WolfSheep.py => wolf_sheep.py} | 0 benchmarks/configurations.py | 8 +- benchmarks/global_benchmark.py | 6 +- 8 files changed, 135 insertions(+), 97 deletions(-) delete mode 100644 benchmarks/Flocking/Flocking.py rename benchmarks/Flocking/{boid.py => flocking.py} (52%) rename benchmarks/Schelling/{Schelling.py => schelling.py} (74%) rename benchmarks/WolfSheep/{WolfSheep.py => wolf_sheep.py} (100%) diff --git a/.gitignore b/.gitignore index 1b04c1237b5..814f6a5daf3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ # Benchmarking -benchmarking/**/*.pickle +benchmarks/**/*.pickle # Byte-compiled / optimized / DLL files __pycache__/ @@ -69,6 +69,7 @@ target/ # PyCharm environment files .idea/ +*.pclprof # VSCode environment files .vscode/ @@ -88,3 +89,4 @@ pythonenv*/ # JS dependencies mesa/visualization/templates/external/ mesa/visualization/templates/js/external/ + diff --git a/benchmarks/Flocking/Flocking.py b/benchmarks/Flocking/Flocking.py deleted file mode 100644 index 4a5e8529cdd..00000000000 --- a/benchmarks/Flocking/Flocking.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -Flockers -============================================================= -A Mesa implementation of Craig Reynolds's Boids flocker model. -Uses numpy arrays to represent vectors. -""" - -import numpy as np - -from mesa import Model -from mesa.space import ContinuousSpace -from mesa.time import RandomActivation - -from .boid import Boid - - -class BoidFlockers(Model): - """ - Flocker model class. Handles agent creation, placement and scheduling. - """ - - def __init__( - self, - seed, - population, - width, - height, - vision, - speed=1, - separation=1, - cohere=0.03, - separate=0.015, - match=0.05, - ): - """ - Create a new Flockers model. - - Args: - population: Number of Boids - width, height: Size of the space. - speed: How fast should the Boids move. - vision: How far around should each Boid look for its neighbors - separation: What's the minimum distance each Boid will attempt to - keep from any other - cohere, separate, match: factors for the relative importance of - the three drives.""" - super().__init__(seed=seed) - self.population = population - self.vision = vision - self.speed = speed - self.separation = separation - self.schedule = RandomActivation(self) - self.space = ContinuousSpace(width, height, True) - self.factors = {"cohere": cohere, "separate": separate, "match": match} - self.make_agents() - - def make_agents(self): - """ - Create self.population agents, with random positions and starting headings. - """ - for i in range(self.population): - x = self.random.random() * self.space.x_max - y = self.random.random() * self.space.y_max - pos = np.array((x, y)) - velocity = np.random.random(2) * 2 - 1 - boid = Boid( - i, - self, - pos, - self.speed, - velocity, - self.vision, - self.separation, - **self.factors, - ) - self.space.place_agent(boid, pos) - self.schedule.add(boid) - - def step(self): - self.schedule.step() diff --git a/benchmarks/Flocking/boid.py b/benchmarks/Flocking/flocking.py similarity index 52% rename from benchmarks/Flocking/boid.py rename to benchmarks/Flocking/flocking.py index 598ea32b84f..0331d356e69 100644 --- a/benchmarks/Flocking/boid.py +++ b/benchmarks/Flocking/flocking.py @@ -1,6 +1,15 @@ +""" +Flockers +============================================================= +A Mesa implementation of Craig Reynolds's Boids flocker model. +Uses numpy arrays to represent vectors. +""" + import numpy as np -from mesa import Agent +from mesa import Agent, Model +from mesa.space import ContinuousSpace +from mesa.time import RandomActivation class Boid(Agent): @@ -79,3 +88,82 @@ def step(self): self.velocity /= np.linalg.norm(self.velocity) new_pos = self.pos + self.velocity * self.speed self.model.space.move_agent(self, new_pos) + + +class BoidFlockers(Model): + """ + Flocker model class. Handles agent creation, placement and scheduling. + """ + + def __init__( + self, + seed, + population, + width, + height, + vision, + speed=1, + separation=1, + cohere=0.03, + separate=0.015, + match=0.05, + ): + """ + Create a new Flockers model. + + Args: + population: Number of Boids + width, height: Size of the space. + speed: How fast should the Boids move. + vision: How far around should each Boid look for its neighbors + separation: What's the minimum distance each Boid will attempt to + keep from any other + cohere, separate, match: factors for the relative importance of + the three drives.""" + super().__init__(seed=seed) + self.population = population + self.vision = vision + self.speed = speed + self.separation = separation + self.schedule = RandomActivation(self) + self.space = ContinuousSpace(width, height, True) + self.factors = {"cohere": cohere, "separate": separate, "match": match} + self.make_agents() + + def make_agents(self): + """ + Create self.population agents, with random positions and starting headings. + """ + for i in range(self.population): + x = self.random.random() * self.space.x_max + y = self.random.random() * self.space.y_max + pos = np.array((x, y)) + velocity = np.random.random(2) * 2 - 1 + boid = Boid( + i, + self, + pos, + self.speed, + velocity, + self.vision, + self.separation, + **self.factors, + ) + self.space.place_agent(boid, pos) + self.schedule.add(boid) + + def step(self): + self.schedule.step() + + +if __name__ == "__main__": + import time + + # model = BoidFlockers(15, 200, 100, 100, 5) + model = BoidFlockers(15, 400, 100, 100, 15) + + start_time = time.perf_counter() + for _ in range(100): + model.step() + + print(time.perf_counter() - start_time) diff --git a/benchmarks/Schelling/Schelling.py b/benchmarks/Schelling/schelling.py similarity index 74% rename from benchmarks/Schelling/Schelling.py rename to benchmarks/Schelling/schelling.py index eb1379edbfa..624dd0b3b9a 100644 --- a/benchmarks/Schelling/Schelling.py +++ b/benchmarks/Schelling/schelling.py @@ -1,4 +1,3 @@ -import random from mesa import Agent, Model from mesa.space import SingleGrid @@ -10,7 +9,7 @@ class SchellingAgent(Agent): Schelling segregation agent """ - def __init__(self, pos, model, agent_type): + def __init__(self, unique_id, model, agent_type): """ Create a new Schelling agent. Args: @@ -18,14 +17,12 @@ def __init__(self, pos, model, agent_type): x, y: Agent initial location. agent_type: Indicator for the agent's type (minority=1, majority=0) """ - super().__init__(pos, model) - self.pos = pos + super().__init__(unique_id, model) self.type = agent_type def step(self): similar = 0 - r = self.model.radius - for neighbor in self.model.grid.iter_neighbors(self.pos, moore=True, radius=r): + for neighbor in self.model.grid.iter_neighbors(self.pos, moore=True, radius=self.model.radius): if neighbor.type == self.type: similar += 1 @@ -36,7 +33,7 @@ def step(self): self.model.happy += 1 -class SchellingModel(Model): +class Schelling(Model): """ Model class for the Schelling segregation model. """ @@ -63,9 +60,9 @@ def __init__( # the coordinates of a cell as well as # its contents. (coord_iter) for _cont, pos in self.grid.coord_iter(): - if random.random() < self.density: # noqa: S311 - agent_type = 1 if random.random() < self.minority_pc else 0 # noqa: S311 - agent = SchellingAgent(pos, self, agent_type) + if self.random.random() < self.density: + agent_type = 1 if self.random.random() < self.minority_pc else 0 + agent = SchellingAgent(self.next_id(), self, agent_type) self.grid.place_agent(agent, pos) self.schedule.add(agent) @@ -75,3 +72,16 @@ def step(self): """ self.happy = 0 # Reset counter of happy agents self.schedule.step() + + +if __name__ == "__main__": + import time + + # model = Schelling(15, 40, 40, 3, 1, 0.625) + model = Schelling(15, 100, 100, 8, 2, 0.8) + + start_time = time.perf_counter() + for _ in range(100): + model.step() + + print(time.perf_counter() - start_time) diff --git a/benchmarks/WolfSheep/__init__.py b/benchmarks/WolfSheep/__init__.py index e69de29bb2d..98b1e9fdfed 100644 --- a/benchmarks/WolfSheep/__init__.py +++ b/benchmarks/WolfSheep/__init__.py @@ -0,0 +1,14 @@ +from .wolf_sheep import WolfSheep + +if __name__ == "__main__": + # for profiling this benchmark model + import time + + # model = WolfSheep(15, 25, 25, 60, 40, 0.2, 0.1, 20) + model = WolfSheep(15, 100, 100, 1000, 500, 0.4, 0.2, 20) + + start_time = time.perf_counter() + for _ in range(100): + model.step() + + print(time.perf_counter() - start_time) diff --git a/benchmarks/WolfSheep/WolfSheep.py b/benchmarks/WolfSheep/wolf_sheep.py similarity index 100% rename from benchmarks/WolfSheep/WolfSheep.py rename to benchmarks/WolfSheep/wolf_sheep.py diff --git a/benchmarks/configurations.py b/benchmarks/configurations.py index ab8960408d4..c42f69adfb6 100644 --- a/benchmarks/configurations.py +++ b/benchmarks/configurations.py @@ -1,10 +1,10 @@ -from Flocking.Flocking import BoidFlockers -from Schelling.Schelling import SchellingModel -from WolfSheep.WolfSheep import WolfSheep +from Flocking.flocking import BoidFlockers +from Schelling.schelling import Schelling +from WolfSheep.wolf_sheep import WolfSheep configurations = { # Schelling Model Configurations - SchellingModel: { + Schelling: { "small": { "seeds": 50, "replications": 5, diff --git a/benchmarks/global_benchmark.py b/benchmarks/global_benchmark.py index 9547d94a674..0f8d5379732 100644 --- a/benchmarks/global_benchmark.py +++ b/benchmarks/global_benchmark.py @@ -5,6 +5,10 @@ import time import timeit +# making sure we use this version of mesa and not one +# also installed in site_packages or so. +sys.path.insert(0, os.path.abspath("..")) + from configurations import configurations @@ -56,7 +60,7 @@ def run_experiments(model_class, config): mean_run = sum(results[1]) / len(results[1]) print( - f"{time.strftime("%H:%M:%S", time.localtime())} {model.__name__:<14} ({size}) timings: Init {mean_init:.5f} s; Run {mean_run:.4f} s" + f"{time.strftime('%H:%M:%S', time.localtime())} {model.__name__:<14} ({size}) timings: Init {mean_init:.5f} s; Run {mean_run:.4f} s" ) results_dict[model, size] = results