From 8973d568925ac5f16f829d6bf3fc88158e412d4c Mon Sep 17 00:00:00 2001 From: Ewout ter Hoeven Date: Sat, 20 Jan 2024 03:26:03 +0100 Subject: [PATCH] Add performance benchmarking scripts (#1979) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * benchmarks: Upload initial models from JuliaDynamics https://github.com/JuliaDynamics/ABMFrameworksComparison/tree/5551d7abf1611d377b3b32346c7774f176af4c65 * benchmarks: Add dictionary with configurations * benchmarks: Update configurations, use relative imports * benchmarks: Add single script to run all benchmarks * benchmarks: Update configurations Few less replications, some more seeds. Every benchmark now takes between 10 and 20 seconds (on my machine). * benchmarks: Add generated pickle files to gitignore That allows switching branches without benchmarks results disappearing * benchmarks: Update global script to calculate and save stuff Prints some stuff when running and saves a pickle file after running. * benchmarks: Write a script to statistically compare runs - The bootstrap_speedup_confidence_interval function calculates the mean speedup and its confidence interval using bootstrapping, which is more suitable for paired data. - The mean speedup and confidence interval are calculated for both initialization and run times. - Positive values indicate an increase in time (longer duration), and negative values indicate a decrease (shorter duration). - The results are displayed in a DataFrame with the percentage changes and their confidence intervals. * benchmarks: Remove seperate benchmark scripts * benchmarks: Black and ruff * benchmarks: Use old f-string formatting to work with Python 3.11 and older * benchmarks: Add fancy colored performance indicators 🟢🔴🔵 If the 95% confidence interval is: - fully below -3%: 🟢 - fully above +3%: 🔴 - else: 🔵 * Fix typos caught by codespell * Apply ruff format benchmarks/ * Apply ruff --fix benchmarks/ * Apply ruff --fix --unsafe-fixes benchmarks/ * Apply manual Ruff fixes * codecov: Ignore benchmarks/ --------- Co-authored-by: rht --- .gitignore | 3 + benchmarks/Flocking/Flocking.py | 80 ++++++++++++++++++++ benchmarks/Flocking/__init__.py | 0 benchmarks/Flocking/boid.py | 81 ++++++++++++++++++++ benchmarks/Schelling/Schelling.py | 77 +++++++++++++++++++ benchmarks/Schelling/__init__.py | 0 benchmarks/WolfSheep/WolfSheep.py | 103 ++++++++++++++++++++++++++ benchmarks/WolfSheep/__init__.py | 0 benchmarks/WolfSheep/agents.py | 111 ++++++++++++++++++++++++++++ benchmarks/WolfSheep/random_walk.py | 37 ++++++++++ benchmarks/compare_timings.py | 87 ++++++++++++++++++++++ benchmarks/configurations.py | 84 +++++++++++++++++++++ benchmarks/global_benchmark.py | 74 +++++++++++++++++++ codecov.yaml | 3 +- 14 files changed, 739 insertions(+), 1 deletion(-) create mode 100644 benchmarks/Flocking/Flocking.py create mode 100644 benchmarks/Flocking/__init__.py create mode 100644 benchmarks/Flocking/boid.py create mode 100644 benchmarks/Schelling/Schelling.py create mode 100644 benchmarks/Schelling/__init__.py create mode 100644 benchmarks/WolfSheep/WolfSheep.py create mode 100644 benchmarks/WolfSheep/__init__.py create mode 100644 benchmarks/WolfSheep/agents.py create mode 100644 benchmarks/WolfSheep/random_walk.py create mode 100644 benchmarks/compare_timings.py create mode 100644 benchmarks/configurations.py create mode 100644 benchmarks/global_benchmark.py diff --git a/.gitignore b/.gitignore index e2819243943..1b04c1237b5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# Benchmarking +benchmarking/**/*.pickle + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/benchmarks/Flocking/Flocking.py b/benchmarks/Flocking/Flocking.py new file mode 100644 index 00000000000..4a5e8529cdd --- /dev/null +++ b/benchmarks/Flocking/Flocking.py @@ -0,0 +1,80 @@ +""" +Flockers +============================================================= +A Mesa implementation of Craig Reynolds's Boids flocker model. +Uses numpy arrays to represent vectors. +""" + +import numpy as np + +from mesa import Model +from mesa.space import ContinuousSpace +from mesa.time import RandomActivation + +from .boid import Boid + + +class BoidFlockers(Model): + """ + Flocker model class. Handles agent creation, placement and scheduling. + """ + + def __init__( + self, + seed, + population, + width, + height, + vision, + speed=1, + separation=1, + cohere=0.03, + separate=0.015, + match=0.05, + ): + """ + Create a new Flockers model. + + Args: + population: Number of Boids + width, height: Size of the space. + speed: How fast should the Boids move. + vision: How far around should each Boid look for its neighbors + separation: What's the minimum distance each Boid will attempt to + keep from any other + cohere, separate, match: factors for the relative importance of + the three drives.""" + super().__init__(seed=seed) + self.population = population + self.vision = vision + self.speed = speed + self.separation = separation + self.schedule = RandomActivation(self) + self.space = ContinuousSpace(width, height, True) + self.factors = {"cohere": cohere, "separate": separate, "match": match} + self.make_agents() + + def make_agents(self): + """ + Create self.population agents, with random positions and starting headings. + """ + for i in range(self.population): + x = self.random.random() * self.space.x_max + y = self.random.random() * self.space.y_max + pos = np.array((x, y)) + velocity = np.random.random(2) * 2 - 1 + boid = Boid( + i, + self, + pos, + self.speed, + velocity, + self.vision, + self.separation, + **self.factors, + ) + self.space.place_agent(boid, pos) + self.schedule.add(boid) + + def step(self): + self.schedule.step() diff --git a/benchmarks/Flocking/__init__.py b/benchmarks/Flocking/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/benchmarks/Flocking/boid.py b/benchmarks/Flocking/boid.py new file mode 100644 index 00000000000..598ea32b84f --- /dev/null +++ b/benchmarks/Flocking/boid.py @@ -0,0 +1,81 @@ +import numpy as np + +from mesa import Agent + + +class Boid(Agent): + """ + A Boid-style flocker agent. + + The agent follows three behaviors to flock: + - Cohesion: steering towards neighboring agents. + - Separation: avoiding getting too close to any other agent. + - Alignment: try to fly in the same direction as the neighbors. + + Boids have a vision that defines the radius in which they look for their + neighbors to flock with. Their speed (a scalar) and velocity (a vector) + define their movement. Separation is their desired minimum distance from + any other Boid. + """ + + def __init__( + self, + unique_id, + model, + pos, + speed, + velocity, + vision, + separation, + cohere=0.03, + separate=0.015, + match=0.05, + ): + """ + Create a new Boid flocker agent. + + Args: + unique_id: Unique agent identifier. + pos: Starting position + speed: Distance to move per step. + heading: numpy vector for the Boid's direction of movement. + vision: Radius to look around for nearby Boids. + separation: Minimum distance to maintain from other Boids. + cohere: the relative importance of matching neighbors' positions + separate: the relative importance of avoiding close neighbors + match: the relative importance of matching neighbors' headings + + """ + super().__init__(unique_id, model) + self.pos = np.array(pos) + self.speed = speed + self.velocity = velocity + self.vision = vision + self.separation = separation + self.cohere_factor = cohere + self.separate_factor = separate + self.match_factor = match + + def step(self): + """ + Get the Boid's neighbors, compute the new vector, and move accordingly. + """ + + neighbors = self.model.space.get_neighbors(self.pos, self.vision, False) + n = 0 + match_vector, separation_vector, cohere = np.zeros((3, 2)) + for neighbor in neighbors: + n += 1 + heading = self.model.space.get_heading(self.pos, neighbor.pos) + cohere += heading + if self.model.space.get_distance(self.pos, neighbor.pos) < self.separation: + separation_vector -= heading + match_vector += neighbor.velocity + n = max(n, 1) + cohere = cohere * self.cohere_factor + separation_vector = separation_vector * self.separate_factor + match_vector = match_vector * self.match_factor + self.velocity += (cohere + separation_vector + match_vector) / n + self.velocity /= np.linalg.norm(self.velocity) + new_pos = self.pos + self.velocity * self.speed + self.model.space.move_agent(self, new_pos) diff --git a/benchmarks/Schelling/Schelling.py b/benchmarks/Schelling/Schelling.py new file mode 100644 index 00000000000..eb1379edbfa --- /dev/null +++ b/benchmarks/Schelling/Schelling.py @@ -0,0 +1,77 @@ +import random + +from mesa import Agent, Model +from mesa.space import SingleGrid +from mesa.time import RandomActivation + + +class SchellingAgent(Agent): + """ + Schelling segregation agent + """ + + def __init__(self, pos, model, agent_type): + """ + Create a new Schelling agent. + Args: + unique_id: Unique identifier for the agent. + x, y: Agent initial location. + agent_type: Indicator for the agent's type (minority=1, majority=0) + """ + super().__init__(pos, model) + self.pos = pos + self.type = agent_type + + def step(self): + similar = 0 + r = self.model.radius + for neighbor in self.model.grid.iter_neighbors(self.pos, moore=True, radius=r): + if neighbor.type == self.type: + similar += 1 + + # If unhappy, move: + if similar < self.model.homophily: + self.model.grid.move_to_empty(self) + else: + self.model.happy += 1 + + +class SchellingModel(Model): + """ + Model class for the Schelling segregation model. + """ + + def __init__( + self, seed, height, width, homophily, radius, density, minority_pc=0.5 + ): + """ """ + super().__init__(seed=seed) + self.height = height + self.width = width + self.density = density + self.minority_pc = minority_pc + self.homophily = homophily + self.radius = radius + + self.schedule = RandomActivation(self) + self.grid = SingleGrid(height, width, torus=True) + + self.happy = 0 + + # Set up agents + # We use a grid iterator that returns + # the coordinates of a cell as well as + # its contents. (coord_iter) + for _cont, pos in self.grid.coord_iter(): + if random.random() < self.density: # noqa: S311 + agent_type = 1 if random.random() < self.minority_pc else 0 # noqa: S311 + agent = SchellingAgent(pos, self, agent_type) + self.grid.place_agent(agent, pos) + self.schedule.add(agent) + + def step(self): + """ + Run one step of the model. + """ + self.happy = 0 # Reset counter of happy agents + self.schedule.step() diff --git a/benchmarks/Schelling/__init__.py b/benchmarks/Schelling/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/benchmarks/WolfSheep/WolfSheep.py b/benchmarks/WolfSheep/WolfSheep.py new file mode 100644 index 00000000000..b08afbc57ba --- /dev/null +++ b/benchmarks/WolfSheep/WolfSheep.py @@ -0,0 +1,103 @@ +""" +Wolf-Sheep Predation Model +================================ + +Replication of the model found in NetLogo: + Wilensky, U. (1997). NetLogo Wolf Sheep Predation model. + http://ccl.northwestern.edu/netlogo/models/WolfSheepPredation. + Center for Connected Learning and Computer-Based Modeling, + Northwestern University, Evanston, IL. +""" + +import mesa +from mesa.space import MultiGrid +from mesa.time import RandomActivationByType + +from .agents import GrassPatch, Sheep, Wolf + + +class WolfSheep(mesa.Model): + """ + Wolf-Sheep Predation Model + + A model for simulating wolf and sheep (predator-prey) ecosystem modelling. + """ + + def __init__( + self, + seed, + height, + width, + initial_sheep, + initial_wolves, + sheep_reproduce, + wolf_reproduce, + grass_regrowth_time, + wolf_gain_from_food=13, + sheep_gain_from_food=5, + ): + """ + Create a new Wolf-Sheep model with the given parameters. + + Args: + initial_sheep: Number of sheep to start with + initial_wolves: Number of wolves to start with + sheep_reproduce: Probability of each sheep reproducing each step + wolf_reproduce: Probability of each wolf reproducing each step + wolf_gain_from_food: Energy a wolf gains from eating a sheep + grass: Whether to have the sheep eat grass for energy + grass_regrowth_time: How long it takes for a grass patch to regrow + once it is eaten + sheep_gain_from_food: Energy sheep gain from grass, if enabled. + """ + super().__init__(seed=seed) + # Set parameters + self.height = height + self.width = width + self.initial_sheep = initial_sheep + self.initial_wolves = initial_wolves + self.sheep_reproduce = sheep_reproduce + self.wolf_reproduce = wolf_reproduce + self.wolf_gain_from_food = wolf_gain_from_food + self.grass_regrowth_time = grass_regrowth_time + self.sheep_gain_from_food = sheep_gain_from_food + + self.schedule = RandomActivationByType(self) + self.grid = MultiGrid(self.height, self.width, torus=False) + + # Create sheep: + for _i in range(self.initial_sheep): + pos = ( + self.random.randrange(self.width), + self.random.randrange(self.height), + ) + energy = self.random.randrange(2 * self.sheep_gain_from_food) + sheep = Sheep(self.next_id(), pos, self, True, energy) + self.grid.place_agent(sheep, pos) + self.schedule.add(sheep) + + # Create wolves + for _i in range(self.initial_wolves): + pos = ( + self.random.randrange(self.width), + self.random.randrange(self.height), + ) + energy = self.random.randrange(2 * self.wolf_gain_from_food) + wolf = Wolf(self.next_id(), pos, self, True, energy) + self.grid.place_agent(wolf, pos) + self.schedule.add(wolf) + + # Create grass patches + possibly_fully_grown = [True, False] + for _agent, pos in self.grid.coord_iter(): + fully_grown = self.random.choice(possibly_fully_grown) + if fully_grown: + countdown = self.grass_regrowth_time + else: + countdown = self.random.randrange(self.grass_regrowth_time) + patch = GrassPatch(self.next_id(), pos, self, fully_grown, countdown) + self.grid.place_agent(patch, pos) + self.schedule.add(patch) + + def step(self): + self.schedule.step() diff --git a/benchmarks/WolfSheep/__init__.py b/benchmarks/WolfSheep/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/benchmarks/WolfSheep/agents.py b/benchmarks/WolfSheep/agents.py new file mode 100644 index 00000000000..9ad9f7652cc --- /dev/null +++ b/benchmarks/WolfSheep/agents.py @@ -0,0 +1,111 @@ +from mesa import Agent + +from .random_walk import RandomWalker + + +class Sheep(RandomWalker): + """ + A sheep that walks around, reproduces (asexually) and gets eaten. + + The init is the same as the RandomWalker. + """ + + def __init__(self, unique_id, pos, model, moore, energy=None): + super().__init__(unique_id, pos, model, moore=moore) + self.energy = energy + + def step(self): + """ + A model step. Move, then eat grass and reproduce. + """ + self.random_move() + + # Reduce energy + self.energy -= 1 + + # If there is grass available, eat it + this_cell = self.model.grid.get_cell_list_contents(self.pos) + grass_patch = next(obj for obj in this_cell if isinstance(obj, GrassPatch)) + if grass_patch.fully_grown: + self.energy += self.model.sheep_gain_from_food + grass_patch.fully_grown = False + + # Death + if self.energy < 0: + self.model.grid.remove_agent(self) + self.model.schedule.remove(self) + elif self.random.random() < self.model.sheep_reproduce: + # Create a new sheep: + self.energy /= 2 + lamb = Sheep( + self.model.next_id(), self.pos, self.model, self.moore, self.energy + ) + self.model.grid.place_agent(lamb, self.pos) + self.model.schedule.add(lamb) + + +class Wolf(RandomWalker): + """ + A wolf that walks around, reproduces (asexually) and eats sheep. + """ + + def __init__(self, unique_id, pos, model, moore, energy=None): + super().__init__(unique_id, pos, model, moore=moore) + self.energy = energy + + def step(self): + self.random_move() + self.energy -= 1 + + # If there are sheep present, eat one + x, y = self.pos + this_cell = self.model.grid.get_cell_list_contents([self.pos]) + sheep = [obj for obj in this_cell if isinstance(obj, Sheep)] + if len(sheep) > 0: + sheep_to_eat = self.random.choice(sheep) + self.energy += self.model.wolf_gain_from_food + + # Kill the sheep + self.model.grid.remove_agent(sheep_to_eat) + self.model.schedule.remove(sheep_to_eat) + + # Death or reproduction + if self.energy < 0: + self.model.grid.remove_agent(self) + self.model.schedule.remove(self) + elif self.random.random() < self.model.wolf_reproduce: + # Create a new wolf cub + self.energy /= 2 + cub = Wolf( + self.model.next_id(), self.pos, self.model, self.moore, self.energy + ) + self.model.grid.place_agent(cub, cub.pos) + self.model.schedule.add(cub) + + +class GrassPatch(Agent): + """ + A patch of grass that grows at a fixed rate and it is eaten by sheep + """ + + def __init__(self, unique_id, pos, model, fully_grown, countdown): + """ + Creates a new patch of grass + + Args: + grown: (boolean) Whether the patch of grass is fully grown or not + countdown: Time for the patch of grass to be fully grown again + """ + super().__init__(unique_id, model) + self.fully_grown = fully_grown + self.countdown = countdown + self.pos = pos + + def step(self): + if not self.fully_grown: + if self.countdown <= 0: + # Set as fully grown + self.fully_grown = True + self.countdown = self.model.grass_regrowth_time + else: + self.countdown -= 1 diff --git a/benchmarks/WolfSheep/random_walk.py b/benchmarks/WolfSheep/random_walk.py new file mode 100644 index 00000000000..e55a1f7ed2b --- /dev/null +++ b/benchmarks/WolfSheep/random_walk.py @@ -0,0 +1,37 @@ +""" +Generalized behavior for random walking, one grid cell at a time. +""" + +from mesa import Agent + + +class RandomWalker(Agent): + """ + Class implementing random walker methods in a generalized manner. + + Not intended to be used on its own, but to inherit its methods to multiple + other agents. + + """ + + def __init__(self, unique_id, pos, model, moore=True): + """ + grid: The MultiGrid object in which the agent lives. + x: The agent's current x coordinate + y: The agent's current y coordinate + moore: If True, may move in all 8 directions. + Otherwise, only up, down, left, right. + """ + super().__init__(unique_id, model) + self.pos = pos + self.moore = moore + + def random_move(self): + """ + Step one cell in any allowable direction. + """ + # Pick the next cell from the adjacent cells. + next_moves = self.model.grid.get_neighborhood(self.pos, self.moore, True) + next_move = self.random.choice(next_moves) + # Now move: + self.model.grid.move_agent(self, next_move) diff --git a/benchmarks/compare_timings.py b/benchmarks/compare_timings.py new file mode 100644 index 00000000000..d6f1c11cf89 --- /dev/null +++ b/benchmarks/compare_timings.py @@ -0,0 +1,87 @@ +import pickle + +import numpy as np +import pandas as pd + +filename1 = "timings_1" +filename2 = "timings_2" + +with open(f"{filename1}.pickle", "rb") as handle: + timings_1 = pickle.load(handle) # noqa: S301 + +with open(f"{filename2}.pickle", "rb") as handle: + timings_2 = pickle.load(handle) # noqa: S301 + + +# Function to calculate the percentage change and perform bootstrap to estimate the confidence interval +def bootstrap_percentage_change_confidence_interval(data1, data2, n=1000): + change_samples = [] + for _ in range(n): + sampled_indices = np.random.choice( + range(len(data1)), size=len(data1), replace=True + ) + sampled_data1 = np.array(data1)[sampled_indices] + sampled_data2 = np.array(data2)[sampled_indices] + change = 100 * (sampled_data2 - sampled_data1) / sampled_data1 + change_samples.append(np.mean(change)) + lower, upper = np.percentile(change_samples, [2.5, 97.5]) + return np.mean(change_samples), lower, upper + + +# DataFrame to store the results +results_df = pd.DataFrame() + + +# Function to determine the emoji based on change and confidence interval +def performance_emoji(lower, upper): + if upper < -3: + return "🟢" # Emoji for faster performance + elif lower > 3: + return "🔴" # Emoji for slower performance + else: + return "🔵" # Emoji for insignificant change + + +# Iterate over the models and sizes, perform analysis, and populate the DataFrame +for model, size in timings_1: + model_name = model.__name__ + + # Calculate percentage change and confidence interval for init times + ( + init_change, + init_lower, + init_upper, + ) = bootstrap_percentage_change_confidence_interval( + timings_1[(model, size)][0], timings_2[(model, size)][0] + ) + init_emoji = performance_emoji(init_lower, init_upper) + init_summary = ( + f"{init_emoji} {init_change:+.1f}% [{init_lower:+.1f}%, {init_upper:+.1f}%]" + ) + + # Calculate percentage change and confidence interval for run times + run_change, run_lower, run_upper = bootstrap_percentage_change_confidence_interval( + timings_1[(model, size)][1], timings_2[(model, size)][1] + ) + run_emoji = performance_emoji(run_lower, run_upper) + run_summary = ( + f"{run_emoji} {run_change:+.1f}% [{run_lower:+.1f}%, {run_upper:+.1f}%]" + ) + + # Append results to DataFrame + row = pd.DataFrame( + { + "Model": [model_name], + "Size": [size], + "Init time [95% CI]": [init_summary], + "Run time [95% CI]": [run_summary], + } + ) + + results_df = pd.concat([results_df, row], ignore_index=True) + +# Convert DataFrame to markdown with specified alignments +markdown_representation = results_df.to_markdown(index=False, tablefmt="github") + +# Display the markdown representation +print(markdown_representation) diff --git a/benchmarks/configurations.py b/benchmarks/configurations.py new file mode 100644 index 00000000000..ab8960408d4 --- /dev/null +++ b/benchmarks/configurations.py @@ -0,0 +1,84 @@ +from Flocking.Flocking import BoidFlockers +from Schelling.Schelling import SchellingModel +from WolfSheep.WolfSheep import WolfSheep + +configurations = { + # Schelling Model Configurations + SchellingModel: { + "small": { + "seeds": 50, + "replications": 5, + "steps": 20, + "parameters": { + "height": 40, + "width": 40, + "homophily": 3, + "radius": 1, + "density": 0.625, + }, + }, + "large": { + "seeds": 10, + "replications": 3, + "steps": 10, + "parameters": { + "height": 100, + "width": 100, + "homophily": 8, + "radius": 2, + "density": 0.8, + }, + }, + }, + # WolfSheep Model Configurations + WolfSheep: { + "small": { + "seeds": 50, + "replications": 5, + "steps": 40, + "parameters": { + "height": 25, + "width": 25, + "initial_sheep": 60, + "initial_wolves": 40, + "sheep_reproduce": 0.2, + "wolf_reproduce": 0.1, + "grass_regrowth_time": 20, + }, + }, + "large": { + "seeds": 10, + "replications": 3, + "steps": 20, + "parameters": { + "height": 100, + "width": 100, + "initial_sheep": 1000, + "initial_wolves": 500, + "sheep_reproduce": 0.4, + "wolf_reproduce": 0.2, + "grass_regrowth_time": 10, + }, + }, + }, + # BoidFlockers Model Configurations + BoidFlockers: { + "small": { + "seeds": 25, + "replications": 3, + "steps": 20, + "parameters": {"population": 200, "width": 100, "height": 100, "vision": 5}, + }, + "large": { + "seeds": 10, + "replications": 3, + "steps": 10, + "parameters": { + "population": 400, + "width": 150, + "height": 150, + "vision": 15, + }, + }, + }, +} diff --git a/benchmarks/global_benchmark.py b/benchmarks/global_benchmark.py new file mode 100644 index 00000000000..9547d94a674 --- /dev/null +++ b/benchmarks/global_benchmark.py @@ -0,0 +1,74 @@ +import gc +import os +import pickle +import sys +import time +import timeit + +from configurations import configurations + + +# Generic function to initialize and run a model +def run_model(model_class, seed, parameters): + start_init = timeit.default_timer() + model = model_class(seed=seed, **parameters) + # time.sleep(0.001) + + end_init_start_run = timeit.default_timer() + + for _ in range(config["steps"]): + model.step() + # time.sleep(0.0001) + end_run = timeit.default_timer() + + return (end_init_start_run - start_init), (end_run - end_init_start_run) + + +# Function to run experiments and save the fastest replication for each seed +def run_experiments(model_class, config): + gc.enable() + sys.path.insert(0, os.path.abspath(".")) + + init_times = [] + run_times = [] + for seed in range(1, config["seeds"] + 1): + fastest_init = float("inf") + fastest_run = float("inf") + for _replication in range(1, config["replications"] + 1): + init_time, run_time = run_model(model_class, seed, config["parameters"]) + if init_time < fastest_init: + fastest_init = init_time + if run_time < fastest_run: + fastest_run = run_time + init_times.append(fastest_init) + run_times.append(fastest_run) + + return init_times, run_times + + +print(f"{time.strftime('%H:%M:%S', time.localtime())} starting benchmarks.") +results_dict = {} +for model, model_config in configurations.items(): + for size, config in model_config.items(): + results = run_experiments(model, config) + + mean_init = sum(results[0]) / len(results[0]) + mean_run = sum(results[1]) / len(results[1]) + + print( + f"{time.strftime("%H:%M:%S", time.localtime())} {model.__name__:<14} ({size}) timings: Init {mean_init:.5f} s; Run {mean_run:.4f} s" + ) + + results_dict[model, size] = results + +# Change this name to anything you like +save_name = "timings" + +i = 1 +while os.path.exists(f"{save_name}_{i}.pickle"): + i += 1 + +with open(f"{save_name}_{i}.pickle", "wb") as handle: + pickle.dump(results_dict, handle, protocol=pickle.HIGHEST_PROTOCOL) + +print(f"Done benchmarking. Saved results to {save_name}_{i}.pickle.") diff --git a/codecov.yaml b/codecov.yaml index 4993daf03ef..ad078b23fc0 100644 --- a/codecov.yaml +++ b/codecov.yaml @@ -7,5 +7,6 @@ coverage: ignore: - "**experimental/" + - "benchmarks/**" -comment: off \ No newline at end of file +comment: off