From d738534cb7b1087379096de46cb15b8f8d8de657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20B=C3=B6hm?= Date: Mon, 27 Nov 2023 14:23:01 +0100 Subject: [PATCH 1/2] Adds synthetic feature projects (#864) --- .../perf_tests/feature_perf_cs_collection.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index 247b7491b..2ef8e4338 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -1309,3 +1309,97 @@ def compile(self) -> None: def recompile(self) -> None: """Recompile the project.""" _do_feature_perf_cs_collection_recompile(self) + + +class SynthFeatureLargeConfigSpace(VProject): + """Synthetic case-study project for testing flow sensitivity.""" + + NAME = 'SynthFeatureLargeConfigSpace' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local="SynthFeatureLargeConfigSpace", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator( + "SynthFeatureLargeConfigSpace" + ) + ), + FeatureSource() + ] + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + return RevisionBinaryMap( + get_local_project_git_path(SynthFeatureLargeConfigSpace.NAME) + ).specify_binary( + "build/bin/LargeConfigSpace", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("6863c78c24", "HEAD") + )[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_SYNTHFEATURELARGECONFIGSPACE" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self) + + +class SynthFeatureRestrictedConfigSpace(VProject): + """Synthetic case-study project for testing flow sensitivity.""" + + NAME = 'SynthFeatureRestrictedConfigSpace' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local="SynthFeatureRestrictedConfigSpace", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator( + "SynthFeatureRestrictedConfigSpace" + ) + ), + FeatureSource() + ] + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + return RevisionBinaryMap( + get_local_project_git_path(SynthFeatureRestrictedConfigSpace.NAME) + ).specify_binary( + "build/bin/RestrictedConfigSpace", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("6863c78c24", "HEAD") + )[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_SYNTHFEATURERESTRICTEDCONFIGSPACE" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self) From 47e32e2c506fa49058631f95bc0e98c86059fe98 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Tue, 28 Nov 2023 11:18:44 +0100 Subject: [PATCH 2/2] Adds config support for revision look ups (#855) --- tests/revision/__init__.py | 0 tests/revision/test_revisions.py | 34 ++++++++++++++++ varats-core/varats/revision/revisions.py | 50 ++++++++++++++++++------ 3 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 tests/revision/__init__.py create mode 100644 tests/revision/test_revisions.py diff --git a/tests/revision/__init__.py b/tests/revision/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/revision/test_revisions.py b/tests/revision/test_revisions.py new file mode 100644 index 000000000..e54c02d97 --- /dev/null +++ b/tests/revision/test_revisions.py @@ -0,0 +1,34 @@ +"""Test revision helper functions.""" + +import unittest + +from tests.helper_utils import run_in_test_environment, UnitTestFixtures +from varats.paper.paper_config import load_paper_config +from varats.revision.revisions import get_processed_revisions_files +from varats.utils.settings import vara_cfg + + +class TestGetProcessedRevisionsFiles(unittest.TestCase): + """Test if the revision look up works correctly.""" + + @run_in_test_environment( + UnitTestFixtures.PAPER_CONFIGS, UnitTestFixtures.RESULT_FILES + ) + def test_config_specific_lookup(self) -> None: + """Check whether the config specific file loading works.""" + vara_cfg()['paper_config']['current_config'] = "test_config_ids" + load_paper_config() + + processed_rev_files_cid_1 = get_processed_revisions_files( + "SynthSAContextSensitivity", config_id=1 + ) + self.assertEqual(len(processed_rev_files_cid_1), 1) + self.assertEqual( + processed_rev_files_cid_1[0].report_filename.config_id, 1 + ) + + processed_rev_files_cid_2 = get_processed_revisions_files( + "SynthSAContextSensitivity", config_id=2 + ) + # there should not be a report for config id 2 + self.assertEqual(len(processed_rev_files_cid_2), 0) diff --git a/varats-core/varats/revision/revisions.py b/varats-core/varats/revision/revisions.py index 04d987d63..2f50f63ea 100644 --- a/varats-core/varats/revision/revisions.py +++ b/varats-core/varats/revision/revisions.py @@ -123,7 +123,8 @@ def __get_files_with_status( experiment_type: tp.Optional[tp.Type["exp_u.VersionExperiment"]] = None, report_type: tp.Optional[tp.Type[BaseReport]] = None, file_name_filter: tp.Callable[[str], bool] = lambda x: False, - only_newest: bool = True + only_newest: bool = True, + config_id: tp.Optional[int] = None ) -> tp.List[ReportFilepath]: """ Find all file paths to result files with given file statuses. @@ -148,7 +149,15 @@ def __get_files_with_status( result_files = __get_result_files_dict( project_name, experiment_type, report_type ) + for value in result_files.values(): + if config_id is not None: + value = [ + x for x in value if x.report_filename.config_id == config_id + ] + if not value: + continue + sorted_res_files = sorted( value, key=lambda x: x.stat().st_mtime, reverse=True ) @@ -168,7 +177,8 @@ def get_all_revisions_files( experiment_type: tp.Optional[tp.Type["exp_u.VersionExperiment"]] = None, report_type: tp.Optional[tp.Type[BaseReport]] = None, file_name_filter: tp.Callable[[str], bool] = lambda x: False, - only_newest: bool = True + only_newest: bool = True, + config_id: tp.Optional[int] = None ) -> tp.List[ReportFilepath]: """ Find all file paths to revision files. @@ -188,8 +198,13 @@ def get_all_revisions_files( a list of file paths to correctly processed revision files """ return __get_files_with_status( - project_name, list(FileStatusExtension.get_physical_file_statuses()), - experiment_type, report_type, file_name_filter, only_newest + project_name=project_name, + file_statuses=list(FileStatusExtension.get_physical_file_statuses()), + experiment_type=experiment_type, + report_type=report_type, + file_name_filter=file_name_filter, + only_newest=only_newest, + config_id=config_id ) @@ -198,7 +213,8 @@ def get_processed_revisions_files( experiment_type: tp.Optional[tp.Type["exp_u.VersionExperiment"]] = None, report_type: tp.Optional[tp.Type[BaseReport]] = None, file_name_filter: tp.Callable[[str], bool] = lambda x: False, - only_newest: bool = True + only_newest: bool = True, + config_id: tp.Optional[int] = None ) -> tp.List[ReportFilepath]: """ Find all file paths to correctly processed revision files. @@ -218,8 +234,13 @@ def get_processed_revisions_files( a list of file paths to correctly processed revision files """ return __get_files_with_status( - project_name, [FileStatusExtension.SUCCESS], experiment_type, - report_type, file_name_filter, only_newest + project_name=project_name, + file_statuses=[FileStatusExtension.SUCCESS], + experiment_type=experiment_type, + report_type=report_type, + file_name_filter=file_name_filter, + only_newest=only_newest, + config_id=config_id ) @@ -228,7 +249,8 @@ def get_failed_revisions_files( experiment_type: tp.Optional[tp.Type["exp_u.VersionExperiment"]] = None, report_type: tp.Optional[tp.Type[BaseReport]] = None, file_name_filter: tp.Callable[[str], bool] = lambda x: False, - only_newest: bool = True + only_newest: bool = True, + config_id: tp.Optional[int] = None ) -> tp.List[ReportFilepath]: """ Find all file paths to failed revision files. @@ -248,9 +270,15 @@ def get_failed_revisions_files( a list of file paths to failed revision files """ return __get_files_with_status( - project_name, - [FileStatusExtension.FAILED, FileStatusExtension.COMPILE_ERROR], - experiment_type, report_type, file_name_filter, only_newest + project_name=project_name, + file_statuses=[ + FileStatusExtension.FAILED, FileStatusExtension.COMPILE_ERROR + ], + experiment_type=experiment_type, + report_type=report_type, + file_name_filter=file_name_filter, + only_newest=only_newest, + config_id=config_id )