diff --git a/varats-core/varats/experiment/workload_util.py b/varats-core/varats/experiment/workload_util.py index 6cf793572..6e0cc5074 100644 --- a/varats-core/varats/experiment/workload_util.py +++ b/varats-core/varats/experiment/workload_util.py @@ -112,7 +112,7 @@ def filter_by_config(prj_cmd: ProjectCommand) -> bool: ] if not workloads: - print( + LOG.debug( f"Could not find any workloads for {project.name}." f"Tags[{requested_workload_tags}]" ) diff --git a/varats/varats/data/databases/feature_perf_precision_database.py b/varats/varats/data/databases/feature_perf_precision_database.py index 24710c3f1..c6262b32d 100644 --- a/varats/varats/data/databases/feature_perf_precision_database.py +++ b/varats/varats/data/databases/feature_perf_precision_database.py @@ -416,7 +416,7 @@ def get_patch_names(case_study: CaseStudy) -> tp.List[str]: def get_regressing_config_ids_gt( project_name: str, case_study: CaseStudy, rev: FullCommitHash, - report_name: str + patch_name: str ) -> tp.Optional[tp.Dict[int, bool]]: """Computes the baseline data, i.e., the config ids where a regression was identified.""" @@ -445,7 +445,7 @@ def get_regressing_config_ids_gt( old_time = time_reports.get_baseline_report() # new_time = time_reports.get_new_report() - new_time = time_reports.get_report_for_patch(report_name) + new_time = time_reports.get_report_for_patch(patch_name) if not new_time: return None @@ -453,6 +453,8 @@ def get_regressing_config_ids_gt( ) == np.mean(new_time.measurements_wall_clock_time): gt[config_id] = False else: + # TODO: fix to use same check as profilers + # TODO: double check ttest handling ttest_res = ttest_ind( old_time.measurements_wall_clock_time, diff --git a/varats/varats/experiments/vara/feature_perf_precision.py b/varats/varats/experiments/vara/feature_perf_precision.py index 71b6b7b60..12c8742f3 100644 --- a/varats/varats/experiments/vara/feature_perf_precision.py +++ b/varats/varats/experiments/vara/feature_perf_precision.py @@ -8,7 +8,7 @@ from time import sleep import benchbuild.extensions as bb_ext -from benchbuild.command import cleanup +from benchbuild.command import cleanup, ProjectCommand from benchbuild.environments.domain.declarative import ContainerImage from benchbuild.utils import actions from benchbuild.utils.actions import StepResult, Clean @@ -51,6 +51,16 @@ REPS = 3 +def perf_prec_workload_commands( + project: VProject, binary: ProjectBinaryWrapper +) -> tp.List[ProjectCommand]: + return workload_commands(project, binary, [ + WorkloadCategory.EXAMPLE + ]) + workload_commands(project, binary, [ + WorkloadCategory.SMALL + ]) + workload_commands(project, binary, [WorkloadCategory.MEDIUM]) + + class AnalysisProjectStepBase(OutputFolderStep): project: VProject @@ -132,8 +142,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult: zip_tmp_dir = tmp_dir / self._file_name with ZippedReportFolder(zip_tmp_dir) as reps_tmp_dir: for rep in range(0, self._reps): - for prj_command in workload_commands( - self.project, self._binary, [WorkloadCategory.EXAMPLE] + for prj_command in perf_prec_workload_commands( + self.project, self._binary ): local_tracefile_path = Path(reps_tmp_dir) / ( f"trace_{prj_command.command.label}_{rep}" @@ -192,9 +202,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult: with tempfile.TemporaryDirectory() as non_nfs_tmp_dir: with ZippedReportFolder(zip_tmp_dir) as reps_tmp_dir: for rep in range(0, self._reps): - for prj_command in workload_commands( - self.project, self._binary, - [WorkloadCategory.EXAMPLE] + for prj_command in perf_prec_workload_commands( + self.project, self._binary ): local_tracefile_path = Path(reps_tmp_dir) / ( f"trace_{prj_command.command.label}_{rep}" @@ -302,8 +311,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult: zip_tmp_dir = tmp_dir / self._file_name with ZippedReportFolder(zip_tmp_dir) as reps_tmp_dir: for rep in range(0, self._reps): - for prj_command in workload_commands( - self.project, self._binary, [WorkloadCategory.EXAMPLE] + for prj_command in perf_prec_workload_commands( + self.project, self._binary ): local_tracefile_path = Path(reps_tmp_dir) / ( f"trace_{prj_command.command.label}_{rep}" @@ -572,8 +581,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult: zip_tmp_dir = tmp_dir / self.__file_name with ZippedReportFolder(zip_tmp_dir) as reps_tmp_dir: for rep in range(0, self.__reps): - for prj_command in workload_commands( - self.project, self.__binary, [WorkloadCategory.EXAMPLE] + for prj_command in perf_prec_workload_commands( + self.project, self.__binary ): time_report_file = Path(reps_tmp_dir) / ( f"baseline_{prj_command.command.label}_{rep}" @@ -723,8 +732,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult: """Runs the binary with the embedded tracing code.""" with local.cwd(local.path(self.project.builddir)): for rep in range(0, self._reps): - for prj_command in workload_commands( - self.project, self._binary, [WorkloadCategory.EXAMPLE] + for prj_command in perf_prec_workload_commands( + self.project, self._binary ): base = Path("/tmp/") fake_tracefile_path = base / ( @@ -789,8 +798,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult: with local.cwd(local.path(self.project.builddir)): with tempfile.TemporaryDirectory() as non_nfs_tmp_dir: for rep in range(0, self._reps): - for prj_command in workload_commands( - self.project, self._binary, [WorkloadCategory.EXAMPLE] + for prj_command in perf_prec_workload_commands( + self.project, self._binary ): base = Path(non_nfs_tmp_dir) fake_tracefile_path = base / ( @@ -874,8 +883,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult: """Runs the binary with the embedded tracing code.""" with local.cwd(local.path(self.project.builddir)): for rep in range(0, self._reps): - for prj_command in workload_commands( - self.project, self._binary, [WorkloadCategory.EXAMPLE] + for prj_command in perf_prec_workload_commands( + self.project, self._binary ): base = Path("/tmp/") fake_tracefile_path = base / ( @@ -1098,8 +1107,8 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult: """Runs the binary with the embedded tracing code.""" with local.cwd(local.path(self.project.builddir)): for rep in range(0, self.__reps): - for prj_command in workload_commands( - self.project, self.__binary, [WorkloadCategory.EXAMPLE] + for prj_command in perf_prec_workload_commands( + self.project, self.__binary ): time_report_file = tmp_dir / ( f"overhead_{prj_command.command.label}_{rep}" diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index 23c542e9e..04f92776e 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -745,19 +745,6 @@ class SynthIPRuntime(VProject): ] WORKLOADS = { - WorkloadSet(WorkloadCategory.EXAMPLE): [ - VCommand( - SourceRoot("SynthIPRuntime") / RSBinary("Runtime"), - "-c", - "<", - "geo-maps/countries-land-1km.geo.json", - ">", - "geo-maps/countries-land-1km.geo.json.compressed", - label="countries-land-1km", - creates=["geo-maps/countries-land-1km.geo.json.compressed"], - requires_all_args={"-c"} - ) - ], WorkloadSet(WorkloadCategory.SMALL): [ VCommand( SourceRoot("SynthIPRuntime") / RSBinary("Runtime"),