diff --git a/.vscode/spellright.dict b/.vscode/spellright.dict index 62a9a81b4..368799648 100644 --- a/.vscode/spellright.dict +++ b/.vscode/spellright.dict @@ -262,4 +262,8 @@ ctx_loader ctx_arg_name ctx_arg_pos dataset +idx +subprocess +usr +± UNPROCESSABLE_ENTITY diff --git a/ci/benchmark.py b/ci/benchmark.py new file mode 100755 index 000000000..7cef2e31e --- /dev/null +++ b/ci/benchmark.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +""" +This file is a script to run bench-marking tests for the FlexMeasures project. +It can be useful to compare two states, and see if performance has improved or got worse. + +Call like this: + +./ci/benchmark.py --iterations 5 --sensor 6 --sensor 7 --start 2023-01-01T05:00:00.000Z --duration P1M +""" + +import subprocess +import time +import isodate +import numpy as np +import tqdm +import click + +# TODO: do not use CLI, but address the underlying functions directly + +from flexmeasures.data.schemas import DurationField, AwareDateTimeField + + +@click.command() +@click.option("--iterations", type=int, default=10) +@click.option("--sensor", "sensors", type=int, multiple=True, required=True) +@click.option( + "--start", + "start", + type=AwareDateTimeField(format="iso"), + required=True, + help="Start queries at this datetime. Follow up with a timezone-aware datetime in ISO 6801 format.", +) +@click.option( + "--duration", + "duration", + type=DurationField(), + default="P30D", + help="Duration of data to be queried, after --start. Follow up with a duration in ISO 6801 format, e.g. PT1H (1 hour) or PT45M (45 minutes).", +) +def benchmark(iterations, sensors, start, duration): + """ + This command queries belief data a couple times and tells you how long that took on average (and the STD). + """ + start_str = str(start).replace(" ", "T") + start_str = isodate.datetime_isoformat(start) + duration_str = isodate.strftime(duration, "P%P") + sensor_ids = " ".join([f"--sensor-id {sensor_id}" for sensor_id in sensors]) + commands = [ + f"flexmeasures show beliefs {sensor_ids} --start {start_str} --duration {duration_str}", + # TODO: add a scheduling operation (wait for result, no queue) + ] + + print("I'll be running:") + print("\n".join(commands)) + print() + print(f"I'll repeat this {iterations} times.") + print() + + timings = {} + + for idx, command in enumerate(commands): + for _ in tqdm.tqdm(range(iterations)): + start = time.perf_counter() + run_command(command) + stop = time.perf_counter() + try: + timings[idx].append(start - stop) + except KeyError: + timings[idx] = [start - stop] + + # Get mean and standard deviation of timings + print(f"All timings are in seconds and iterated {iterations} times") + for idx, timing in timings.items(): + print(f"Command {commands[idx]}: {np.mean(timing)} ± {np.std(timing)}") + + +def run_command(command): + # Run command using subprocess and check for errors but don't print output + try: + subprocess.run(command, shell=True, check=True, stdout=subprocess.DEVNULL) + except subprocess.CalledProcessError as process_error: + print(process_error) + exit(1) + + +if __name__ == "__main__": + benchmark() diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 9f7ec4054..23361a212 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -19,9 +19,11 @@ New features Infrastructure / Support ---------------------- +- Add benchmark script to test speed of the current code base, running some basic commands. [see `PR #787 `_] - Introduce a new one-to-many relation between assets, allowing the definition of an asset's parent (which is also an asset). This hierarchical relationship enables assets to be related in a structured manner. [see `PR #855 `_ and `PR #874 `_] - Introduce a new format for the output of ``Scheduler`` to prepare for multiple outputs [see `PR #879 `_]. + v0.16.1 | October 2, 2023 ============================