Rework workload selection

varats-core/varats/experiment/workload_util.py

@@ -112,7 +112,7 @@ def filter_by_config(prj_cmd: ProjectCommand) -> bool:
     ]
 
     if not workloads:
-        print(
+        LOG.debug(
             f"Could not find any workloads for {project.name}."
             f"Tags[{requested_workload_tags}]"
         )

varats/varats/data/databases/feature_perf_precision_database.py

@@ -416,7 +416,7 @@ def get_patch_names(case_study: CaseStudy) -> tp.List[str]:
 
 def get_regressing_config_ids_gt(
     project_name: str, case_study: CaseStudy, rev: FullCommitHash,
-    report_name: str
+    patch_name: str
 ) -> tp.Optional[tp.Dict[int, bool]]:
     """Computes the baseline data, i.e., the config ids where a regression was
     identified."""
@@ -445,14 +445,16 @@ def get_regressing_config_ids_gt(
 
         old_time = time_reports.get_baseline_report()
         # new_time = time_reports.get_new_report()
-        new_time = time_reports.get_report_for_patch(report_name)
+        new_time = time_reports.get_report_for_patch(patch_name)
         if not new_time:
             return None
 
         if np.mean(old_time.measurements_wall_clock_time
                   ) == np.mean(new_time.measurements_wall_clock_time):
             gt[config_id] = False
         else:
+            # TODO: fix to use same check as profilers
+
             # TODO: double check ttest handling
             ttest_res = ttest_ind(
                 old_time.measurements_wall_clock_time,

varats/varats/experiments/vara/feature_perf_precision.py

@@ -8,7 +8,7 @@
 from time import sleep
 
 import benchbuild.extensions as bb_ext
-from benchbuild.command import cleanup
+from benchbuild.command import cleanup, ProjectCommand
 from benchbuild.environments.domain.declarative import ContainerImage
 from benchbuild.utils import actions
 from benchbuild.utils.actions import StepResult, Clean
@@ -51,6 +51,16 @@
 REPS = 3
 
 
+def perf_prec_workload_commands(
+    project: VProject, binary: ProjectBinaryWrapper
+) -> tp.List[ProjectCommand]:
+    return workload_commands(project, binary, [
+        WorkloadCategory.EXAMPLE
+    ]) + workload_commands(project, binary, [
+        WorkloadCategory.SMALL
+    ]) + workload_commands(project, binary, [WorkloadCategory.MEDIUM])
+
+
 class AnalysisProjectStepBase(OutputFolderStep):
 
     project: VProject
@@ -132,8 +142,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult:
             zip_tmp_dir = tmp_dir / self._file_name
             with ZippedReportFolder(zip_tmp_dir) as reps_tmp_dir:
                 for rep in range(0, self._reps):
-                    for prj_command in workload_commands(
-                        self.project, self._binary, [WorkloadCategory.EXAMPLE]
+                    for prj_command in perf_prec_workload_commands(
+                        self.project, self._binary
                     ):
                         local_tracefile_path = Path(reps_tmp_dir) / (
                             f"trace_{prj_command.command.label}_{rep}"
@@ -192,9 +202,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult:
             with tempfile.TemporaryDirectory() as non_nfs_tmp_dir:
                 with ZippedReportFolder(zip_tmp_dir) as reps_tmp_dir:
                     for rep in range(0, self._reps):
-                        for prj_command in workload_commands(
-                            self.project, self._binary,
-                            [WorkloadCategory.EXAMPLE]
+                        for prj_command in perf_prec_workload_commands(
+                            self.project, self._binary
                         ):
                             local_tracefile_path = Path(reps_tmp_dir) / (
                                 f"trace_{prj_command.command.label}_{rep}"
@@ -302,8 +311,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult:
             zip_tmp_dir = tmp_dir / self._file_name
             with ZippedReportFolder(zip_tmp_dir) as reps_tmp_dir:
                 for rep in range(0, self._reps):
-                    for prj_command in workload_commands(
-                        self.project, self._binary, [WorkloadCategory.EXAMPLE]
+                    for prj_command in perf_prec_workload_commands(
+                        self.project, self._binary
                     ):
                         local_tracefile_path = Path(reps_tmp_dir) / (
                             f"trace_{prj_command.command.label}_{rep}"
@@ -572,8 +581,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult:
             zip_tmp_dir = tmp_dir / self.__file_name
             with ZippedReportFolder(zip_tmp_dir) as reps_tmp_dir:
                 for rep in range(0, self.__reps):
-                    for prj_command in workload_commands(
-                        self.project, self.__binary, [WorkloadCategory.EXAMPLE]
+                    for prj_command in perf_prec_workload_commands(
+                        self.project, self.__binary
                     ):
                         time_report_file = Path(reps_tmp_dir) / (
                             f"baseline_{prj_command.command.label}_{rep}"
@@ -723,8 +732,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult:
         """Runs the binary with the embedded tracing code."""
         with local.cwd(local.path(self.project.builddir)):
             for rep in range(0, self._reps):
-                for prj_command in workload_commands(
-                    self.project, self._binary, [WorkloadCategory.EXAMPLE]
+                for prj_command in perf_prec_workload_commands(
+                    self.project, self._binary
                 ):
                     base = Path("/tmp/")
                     fake_tracefile_path = base / (
@@ -789,8 +798,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult:
         with local.cwd(local.path(self.project.builddir)):
             with tempfile.TemporaryDirectory() as non_nfs_tmp_dir:
                 for rep in range(0, self._reps):
-                    for prj_command in workload_commands(
-                        self.project, self._binary, [WorkloadCategory.EXAMPLE]
+                    for prj_command in perf_prec_workload_commands(
+                        self.project, self._binary
                     ):
                         base = Path(non_nfs_tmp_dir)
                         fake_tracefile_path = base / (
@@ -874,8 +883,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult:
         """Runs the binary with the embedded tracing code."""
         with local.cwd(local.path(self.project.builddir)):
             for rep in range(0, self._reps):
-                for prj_command in workload_commands(
-                    self.project, self._binary, [WorkloadCategory.EXAMPLE]
+                for prj_command in perf_prec_workload_commands(
+                    self.project, self._binary
                 ):
                     base = Path("/tmp/")
                     fake_tracefile_path = base / (
@@ -1098,8 +1107,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult:
         """Runs the binary with the embedded tracing code."""
         with local.cwd(local.path(self.project.builddir)):
             for rep in range(0, self.__reps):
-                for prj_command in workload_commands(
-                    self.project, self.__binary, [WorkloadCategory.EXAMPLE]
+                for prj_command in perf_prec_workload_commands(
+                    self.project, self.__binary
                 ):
                     time_report_file = tmp_dir / (
                         f"overhead_{prj_command.command.label}_{rep}"

varats/varats/projects/perf_tests/feature_perf_cs_collection.py

@@ -745,19 +745,6 @@ class SynthIPRuntime(VProject):
     ]
 
     WORKLOADS = {
-        WorkloadSet(WorkloadCategory.EXAMPLE): [
-            VCommand(
-                SourceRoot("SynthIPRuntime") / RSBinary("Runtime"),
-                "-c",
-                "<",
-                "geo-maps/countries-land-1km.geo.json",
-                ">",
-                "geo-maps/countries-land-1km.geo.json.compressed",
-                label="countries-land-1km",
-                creates=["geo-maps/countries-land-1km.geo.json.compressed"],
-                requires_all_args={"-c"}
-            )
-        ],
         WorkloadSet(WorkloadCategory.SMALL): [
             VCommand(
                 SourceRoot("SynthIPRuntime") / RSBinary("Runtime"),