Reduce printed error when making reports

benchmate/benchmate/warden.py

@@ -177,8 +177,6 @@ def handler(self, sig, frame):
         # and make the program stop
         self.restore_handlers()
 
-        self.stop()
-
     def restore_handlers(self):
         """Restore old signal handlers"""
         if not self.signal_installed:
@@ -189,6 +187,9 @@ def restore_handlers(self):
 
         self.signal_installed = False
 
+        # Cleanup now
+        self.stop()
+
     def maybe_stop(self):
         """Raise the delayed signal if any or restore the old signal handlers"""
 

milabench/commands/__init__.py

@@ -659,7 +659,7 @@ def _argv(self, **_) -> List:
             else ["--multi_gpu"]
         )
 
-        cpu_per_process = self.resolve_argument('--cpus_per_gpu')
+        cpu_per_process = self.pack.resolve_argument('--cpus_per_gpu')
         return [
             # -- Run the command in the right venv
             # This could be inside the SSH Command

milabench/pack.py

@@ -509,7 +509,7 @@ def build_run_plan(self) -> "cmd.Command":
     def resolve_argument(self, name):
         """Resolve as single placeholder argument"""
         placeholder = str(self.config["argv"][name])
-        return self.resolve_argument(placeholder)
+        return self.resolve_placeholder(placeholder)
 
     def resolve_placeholder(self, placeholder):
         """Resolve as single placeholder argument
@@ -521,5 +521,4 @@ def resolve_placeholder(self, placeholder):
 
         """
         from milabench.sizer import resolve_argv
-
         return str(resolve_argv(self, [str(placeholder)])[0])

milabench/report.py

@@ -33,7 +33,7 @@ def _make_row(summary, compare, config, query=None):
         "enabled": is_enabled,
     }
 
-    if not summary:
+    if not summary or summary.get("empty", False):
         return row
 
     # Count not running an enabled benchmark as a failure
@@ -218,7 +218,8 @@ def make_dataframe(summary, compare=None, weights=None, query=None):
                     "n": 0, 
                     "successes": 0,
                     "failures": 0, 
-                    "enabled": weights[key]["enabled"]
+                    "enabled": weights[key]["enabled"],
+                    "empty": True
                 }
 
     if weights is None: