From b17ef1947420045c0f732a4cf9e2a2464b26a83d Mon Sep 17 00:00:00 2001 From: cboillet-dev <84328515+cboillet-dev@users.noreply.github.com> Date: Thu, 8 Sep 2022 17:13:55 +0200 Subject: [PATCH] feat: benchmark for connect-ci (#13) Signed-off-by: Camille Boillet --- benchmark/camelyon/benchmarks.py | 2 +- benchmark/camelyon/workflows.py | 11 ++--------- docker/substrafl-tests/Dockerfile | 2 ++ 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/benchmark/camelyon/benchmarks.py b/benchmark/camelyon/benchmarks.py index 296b5d1b..e2f614c4 100644 --- a/benchmark/camelyon/benchmarks.py +++ b/benchmark/camelyon/benchmarks.py @@ -130,7 +130,7 @@ def main(): results.update(res) # Save results - LOCAL_RESULTS_FILE.write_text(json.dumps(results, sort_keys=True, indent=4)) + LOCAL_RESULTS_FILE.write_text(json.dumps(list(results), sort_keys=True, indent=4)) finally: # Delete the temporary experiment folders at the end of the benchmark shutil.rmtree("local-worker", ignore_errors=True) diff --git a/benchmark/camelyon/workflows.py b/benchmark/camelyon/workflows.py index e4625e56..501c97a2 100644 --- a/benchmark/camelyon/workflows.py +++ b/benchmark/camelyon/workflows.py @@ -126,15 +126,8 @@ def substrafl_fed_avg( else: time.sleep(1) - testtuples = clients[1].list_testtuple(filters={"compute_plan_key": [compute_plan.key]}) - - # Resetting the clients - # BUG: if we don't reset clients, substra tries to find an algo that do not exist - # TODO: investigate - del clients - - # Returning performances - return {k: list(testtuple.test.perfs.values())[0] for k, testtuple in enumerate(testtuples)} + performances = clients[1].get_performances(compute_plan.key) + return performances.dict().values() def torch_fed_avg( diff --git a/docker/substrafl-tests/Dockerfile b/docker/substrafl-tests/Dockerfile index 7c81b66a..f76f5004 100644 --- a/docker/substrafl-tests/Dockerfile +++ b/docker/substrafl-tests/Dockerfile @@ -20,4 +20,6 @@ RUN cd substra && python -m pip install --no-cache-dir -e . RUN cd substrafl && python -m pip install --no-cache-dir -e .[dev] +RUN cd substrafl/benchmark/camelyon && python -m pip install -r requirements.txt + WORKDIR /usr/src/app/substrafl