feat: benchmark for connect-ci (#13)

Signed-off-by: Camille Boillet <[email protected]>

benchmark/camelyon/benchmarks.py

@@ -130,7 +130,7 @@ def main():
             results.update(res)
 
             # Save results
-            LOCAL_RESULTS_FILE.write_text(json.dumps(results, sort_keys=True, indent=4))
+            LOCAL_RESULTS_FILE.write_text(json.dumps(list(results), sort_keys=True, indent=4))
     finally:
         # Delete the temporary experiment folders at the end of the benchmark
         shutil.rmtree("local-worker", ignore_errors=True)

benchmark/camelyon/workflows.py

@@ -126,15 +126,8 @@ def substrafl_fed_avg(
         else:
             time.sleep(1)
 
-    testtuples = clients[1].list_testtuple(filters={"compute_plan_key": [compute_plan.key]})
-
-    # Resetting the clients
-    # BUG: if we don't reset clients, substra tries to find an algo that do not exist
-    # TODO: investigate
-    del clients
-
-    # Returning performances
-    return {k: list(testtuple.test.perfs.values())[0] for k, testtuple in enumerate(testtuples)}
+    performances = clients[1].get_performances(compute_plan.key)
+    return performances.dict().values()
 
 
 def torch_fed_avg(

docker/substrafl-tests/Dockerfile

@@ -20,4 +20,6 @@ RUN cd substra && python -m pip install --no-cache-dir -e .
 
 RUN cd substrafl && python -m pip install --no-cache-dir -e .[dev]
 
+RUN cd substrafl/benchmark/camelyon && python -m pip install -r requirements.txt
+
 WORKDIR /usr/src/app/substrafl