From db6a69b567a2dc5787b9fa44db466495ce74b243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20B=C3=B6hm?= Date: Wed, 27 Sep 2023 12:00:35 +0200 Subject: [PATCH] Adds synthetic case sudies with different implementation patterns. (#840) --- .../SynthIPTemplate_0.case_study | 25 ++ tests/experiment/test_workload_util.py | 37 +++ tests/utils/test_experiment_util.py | 20 ++ .../perf_tests/feature_perf_cs_collection.py | 307 ++++++++++++++++++ 4 files changed, 389 insertions(+) create mode 100644 tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPTemplate_0.case_study diff --git a/tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPTemplate_0.case_study b/tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPTemplate_0.case_study new file mode 100644 index 000000000..28324d99a --- /dev/null +++ b/tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPTemplate_0.case_study @@ -0,0 +1,25 @@ +--- +DocType: CaseStudy +Version: 1 +... +--- +project_name: SynthIPTemplate +stages: +- revisions: + - commit_hash: 793035062810ea3a2d9a10f831cd199fbbb82090 + commit_id: 64 + config_ids: + - 0 + - 1 + - 2 + - 3 + - 4 +version: 0 +... +--- +config_type: PatchConfiguration +0: '["Decompress"]' +1: '["Compress"]' +2: '["Compress", "fastmode", "no_smallmode"]' +3: '["Compress", "no_fastmode", "smallmode"]' +4: '["Compress", "fastmode", "smallmode"]' diff --git a/tests/experiment/test_workload_util.py b/tests/experiment/test_workload_util.py index b8de4db27..e98e56d00 100644 --- a/tests/experiment/test_workload_util.py +++ b/tests/experiment/test_workload_util.py @@ -6,8 +6,14 @@ from benchbuild.source.base import Revision, Variant import varats.experiment.workload_util as wu +from tests.helper_utils import run_in_test_environment, UnitTestFixtures +from varats.paper.paper_config import load_paper_config from varats.projects.c_projects.xz import Xz +from varats.projects.perf_tests.feature_perf_cs_collection import ( + SynthIPTemplate, +) from varats.utils.git_util import ShortCommitHash +from varats.utils.settings import vara_cfg TT = PathToken.make_token(RootRenderer()) @@ -66,6 +72,37 @@ def test_workload_commands_requires(self) -> None: ) self.assertEqual(len(commands), 1) + @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS) + def test_workload_commands_requires_patch(self) -> None: + vara_cfg()['paper_config']['current_config'] = "test_config_ids" + load_paper_config() + + revision = Revision( + SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"), + Variant(SynthIPTemplate.SOURCE[1], "1") + ) + project = SynthIPTemplate(revision=revision) + binary = SynthIPTemplate.binaries_for_revision( + ShortCommitHash("7930350628") + )[0] + workloads = wu.workload_commands(project, binary, []) + self.assertEqual(len(workloads), 2) + + @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS) + def test_workload_commands_requires_patch2(self) -> None: + vara_cfg()['paper_config']['current_config'] = "test_config_ids" + load_paper_config() + + revision = Revision( + SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"), + Variant(SynthIPTemplate.SOURCE[1], "0") + ) + project = SynthIPTemplate(revision=revision) + binary = SynthIPTemplate \ + .binaries_for_revision(ShortCommitHash("7930350628"))[0] + workloads = wu.workload_commands(project, binary, []) + self.assertEqual(len(workloads), 0) + class TestWorkloadFilenames(unittest.TestCase): diff --git a/tests/utils/test_experiment_util.py b/tests/utils/test_experiment_util.py index 90061005b..a6bc93bd1 100644 --- a/tests/utils/test_experiment_util.py +++ b/tests/utils/test_experiment_util.py @@ -24,6 +24,9 @@ from varats.project.project_util import BinaryType, ProjectBinaryWrapper from varats.project.varats_project import VProject from varats.projects.c_projects.xz import Xz +from varats.projects.perf_tests.feature_perf_cs_collection import ( + SynthIPTemplate, +) from varats.report.gnu_time_report import TimeReport from varats.report.report import FileStatusExtension, ReportSpecification from varats.utils.git_util import ShortCommitHash @@ -419,3 +422,20 @@ def test_get_extra_config_options(self) -> None: ) project = Xz(revision=revision) self.assertEqual(EU.get_extra_config_options(project), ["--foo"]) + + @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS) + def test_get_config_patches(self) -> None: + vara_cfg()['paper_config']['current_config'] = "test_config_ids" + load_paper_config() + + revision = Revision( + SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"), + Variant(SynthIPTemplate.SOURCE[1], "4") + ) + project = SynthIPTemplate(revision=revision) + patches = EU.get_config_patches(project) + self.assertEqual(len(patches), 1) + self.assertEqual( + list(patches)[0].feature_tags, + ["Compress", "fastmode", "smallmode"] + ) diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index cea24265b..6ff8db619 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -4,6 +4,7 @@ import benchbuild as bb from benchbuild.command import Command, SourceRoot, WorkloadSet +from benchbuild.source import HTTPMultiple from benchbuild.utils.cmd import make, cmake, mkdir from benchbuild.utils.revision_ranges import RevisionRange from benchbuild.utils.settings import get_number_of_jobs @@ -19,6 +20,7 @@ verify_binaries, ) from varats.project.sources import FeatureSource +from varats.project.varats_command import VCommand from varats.project.varats_project import VProject from varats.utils.git_commands import init_all_submodules, update_all_submodules from varats.utils.git_util import RevisionBinaryMap, ShortCommitHash @@ -394,3 +396,308 @@ def compile(self) -> None: def recompile(self) -> None: """Recompile the project.""" _do_feature_perf_cs_collection_recompile(self) + + +class SynthIPRuntime(VProject): + """Synthetic case-study project for testing flow sensitivity.""" + + NAME = 'SynthIPRuntime' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local="SynthIPRuntime", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("SynthIPRuntime") + ), + HTTPMultiple( + local="geo-maps", + remote={ + "1.0": + "https://github.com/simonepri/geo-maps/releases/" + "download/v0.6.0" + }, + files=["countries-land-1km.geo.json", "countries-land-1m.geo.json"] + ), + FeatureSource() + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.SMALL): [ + VCommand( + SourceRoot("SynthIPRuntime") / RSBinary("Runtime"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1km", + creates=["geo-maps/countries-land-1km.geo.json.compressed"], + requires_all_args={"-c"} + ) + ], + WorkloadSet(WorkloadCategory.MEDIUM): [ + VCommand( + SourceRoot("SynthIPRuntime") / RSBinary("Runtime"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1m", + creates=["geo-maps/countries-land-1m.geo.json.compressed"], + requires_all_args={"-c"} + ) + ], + } + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + return RevisionBinaryMap( + get_local_project_git_path(SynthIPRuntime.NAME) + ).specify_binary( + "build/bin/Runtime", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("4151c42ffe", "master") + )[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_SYNTHIPRUNTIME" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self) + + +class SynthIPTemplate(VProject): + """Synthetic case-study project for testing flow sensitivity.""" + + NAME = 'SynthIPTemplate' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local="SynthIPTemplate", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("SynthIPTemplate") + ), + FeatureSource() + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.SMALL): [ + VCommand( + SourceRoot("SynthIPTemplate") / RSBinary("Template"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1km", + creates=["geo-maps/countries-land-1km.geo.json.compressed"], + requires_all_patch={"Compress"} + ) + ], + WorkloadSet(WorkloadCategory.MEDIUM): [ + VCommand( + SourceRoot("SynthIPTemplate") / RSBinary("Template"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1m", + creates=["geo-maps/countries-land-1m.geo.json.compressed"], + requires_all_patch={"Compress"} + ) + ], + } + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + return RevisionBinaryMap( + get_local_project_git_path(SynthIPTemplate.NAME) + ).specify_binary( + "build/bin/Template", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("4151c42ffe", "master") + )[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_SYNTHIPTEMPLATE" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self) + + +class SynthIPTemplate2(VProject): + """Synthetic case-study project for testing flow sensitivity.""" + + NAME = 'SynthIPTemplate2' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local="SynthIPTemplate2", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("SynthIPTemplate2") + ), + FeatureSource() + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.SMALL): [ + VCommand( + SourceRoot("SynthIPTemplate2") / RSBinary("Template2"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1km", + creates=["geo-maps/countries-land-1km.geo.json.compressed"], + requires_all_patch={"Compress"} + ) + ], + WorkloadSet(WorkloadCategory.MEDIUM): [ + VCommand( + SourceRoot("SynthIPTemplate2") / RSBinary("Template2"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1m", + creates=["geo-maps/countries-land-1m.geo.json.compressed"], + requires_all_patch={"Compress"} + ) + ], + } + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + return RevisionBinaryMap( + get_local_project_git_path(SynthIPTemplate2.NAME) + ).specify_binary( + "build/bin/Template2", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("4151c42ffe", "master") + )[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_SYNTHIPTEMPLATE2" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self) + + +class SynthIPCombined(VProject): + """Synthetic case-study project for testing flow sensitivity.""" + + NAME = 'SynthIPCombined' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local="SynthIPCombined", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("SynthIPCombined") + ), + FeatureSource() + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.SMALL): [ + VCommand( + SourceRoot("SynthIPCombined") / RSBinary("Combined"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1km", + creates=["geo-maps/countries-land-1km.geo.json.compressed"], + requires_all_args={"-c"} + ) + ], + WorkloadSet(WorkloadCategory.MEDIUM): [ + VCommand( + SourceRoot("SynthIPCombined") / RSBinary("Combined"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1m", + creates=["geo-maps/countries-land-1m.geo.json.compressed"], + requires_all_args={"-c"} + ) + ], + } + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + return RevisionBinaryMap( + get_local_project_git_path(SynthIPCombined.NAME) + ).specify_binary( + "build/bin/Combined", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("4151c42ffe", "master") + )[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_SYNTHIPCOMBINED" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self)