Merge branch 'vara-dev' into f-hyteg-project

docs/source/tutorials/configuration_specific_experiments.rst

@@ -21,6 +21,7 @@ One just needs to extend the case-study file of a project with a yaml document t
 .. code-block:: yaml
 
   ---
+  config_type: PlainCommandlineConfiguration
   0: '["--foo", "--bar"]'
   1: '["--foo"]'
   ...

tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPTemplate_0.case_study

@@ -0,0 +1,25 @@
+---
+DocType: CaseStudy
+Version: 1
+...
+---
+project_name: SynthIPTemplate
+stages:
+- revisions:
+  - commit_hash: 793035062810ea3a2d9a10f831cd199fbbb82090
+    commit_id: 64
+    config_ids:
+    - 0
+    - 1
+    - 2
+    - 3
+    - 4
+version: 0
+...
+---
+config_type: PatchConfiguration
+0: '["Decompress"]'
+1: '["Compress"]'
+2: '["Compress", "fastmode", "no_smallmode"]'
+3: '["Compress", "no_fastmode", "smallmode"]'
+4: '["Compress", "fastmode", "smallmode"]'

tests/TEST_INPUTS/paper_configs/test_config_ids/SynthSAContextSensitivity_0.case_study

@@ -14,6 +14,7 @@ stages:
 version: 0
 ...
 ---
+config_type: PlainCommandlineConfiguration
 0: '["--compress", "--mem", "10", "8"]'
 1: '["--compress", "--mem", "300", "8"]'
 ...

tests/TEST_INPUTS/paper_configs/test_config_ids/xz_0.case_study

@@ -10,5 +10,6 @@ stages:
     config_ids: [1]
 version: 0
 ---
+config_type: PlainCommandlineConfiguration
 0: '["--foo", "--bar"]'
 1: '["--foo"]'

tests/experiment/test_workload_util.py

@@ -6,8 +6,14 @@
 from benchbuild.source.base import Revision, Variant
 
 import varats.experiment.workload_util as wu
+from tests.helper_utils import run_in_test_environment, UnitTestFixtures
+from varats.paper.paper_config import load_paper_config
 from varats.projects.c_projects.xz import Xz
+from varats.projects.perf_tests.feature_perf_cs_collection import (
+    SynthIPTemplate,
+)
 from varats.utils.git_util import ShortCommitHash
+from varats.utils.settings import vara_cfg
 
 TT = PathToken.make_token(RootRenderer())
 
@@ -66,6 +72,37 @@ def test_workload_commands_requires(self) -> None:
         )
         self.assertEqual(len(commands), 1)
 
+    @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS)
+    def test_workload_commands_requires_patch(self) -> None:
+        vara_cfg()['paper_config']['current_config'] = "test_config_ids"
+        load_paper_config()
+
+        revision = Revision(
+            SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"),
+            Variant(SynthIPTemplate.SOURCE[1], "1")
+        )
+        project = SynthIPTemplate(revision=revision)
+        binary = SynthIPTemplate.binaries_for_revision(
+            ShortCommitHash("7930350628")
+        )[0]
+        workloads = wu.workload_commands(project, binary, [])
+        self.assertEqual(len(workloads), 2)
+
+    @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS)
+    def test_workload_commands_requires_patch2(self) -> None:
+        vara_cfg()['paper_config']['current_config'] = "test_config_ids"
+        load_paper_config()
+
+        revision = Revision(
+            SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"),
+            Variant(SynthIPTemplate.SOURCE[1], "0")
+        )
+        project = SynthIPTemplate(revision=revision)
+        binary = SynthIPTemplate \
+            .binaries_for_revision(ShortCommitHash("7930350628"))[0]
+        workloads = wu.workload_commands(project, binary, [])
+        self.assertEqual(len(workloads), 0)
+
 
 class TestWorkloadFilenames(unittest.TestCase):
 

tests/paper/test_case_study.py

@@ -48,6 +48,7 @@
     commit_id: 494
 ...
 ---
+config_type: ConfigurationImpl
 0: '{"foo": true, "bar": false, "bazz": "bazz-value", "buzz": "None"}'
 1: '{}'
 2: '{}'

tests/provider/test_patch_provider.py

@@ -184,57 +184,120 @@ def setUpClass(cls) -> None:
                 "Test-ABCD",
                 "",
                 path=Path("test.patch"),
-                tags={"A", "B", "C", "D"}
+                tags={"A", "B", "C", "D"},
+                feature_tags={"F_A", "F_B", "F_C", "F_D"}
             ),
-            Patch("TEST", "Test-A", "", path=Path("test.patch"), tags={"A"}),
-            Patch("TEST", "Test-B", "", path=Path("test.patch"), tags={"B"}),
-            Patch("TEST", "Test-C", "", path=Path("test.patch"), tags={"C"}),
-            Patch("TEST", "Test-D", "", path=Path("test.patch"), tags={"D"}),
             Patch(
-                "TEST", "Test-AB", "", path=Path("test.patch"), tags={"A", "B"}
+                "TEST",
+                "Test-A",
+                "",
+                path=Path("test.patch"),
+                tags={"A"},
+                feature_tags={"F_A"}
+            ),
+            Patch(
+                "TEST",
+                "Test-B",
+                "",
+                path=Path("test.patch"),
+                tags={"B"},
+                feature_tags={"F_B"}
+            ),
+            Patch(
+                "TEST",
+                "Test-C",
+                "",
+                path=Path("test.patch"),
+                tags={"C"},
+                feature_tags={"F_C"}
             ),
             Patch(
-                "TEST", "Test-AC", "", path=Path("test.patch"), tags={"A", "C"}
+                "TEST",
+                "Test-D",
+                "",
+                path=Path("test.patch"),
+                tags={"D"},
+                feature_tags={"F_D"}
+            ),
+            Patch(
+                "TEST",
+                "Test-AB",
+                "",
+                path=Path("test.patch"),
+                tags={"A", "B"},
+                feature_tags={"F_A", "F_B"}
+            ),
+            Patch(
+                "TEST",
+                "Test-AC",
+                "",
+                path=Path("test.patch"),
+                tags={"A", "C"},
+                feature_tags={"F_A", "F_C"}
             ),
             Patch(
-                "TEST", "Test-AD", "", path=Path("test.patch"), tags={"A", "D"}
+                "TEST",
+                "Test-AD",
+                "",
+                path=Path("test.patch"),
+                tags={"A", "D"},
+                feature_tags={"F_A", "F_D"}
             ),
             Patch(
-                "TEST", "Test-BC", "", path=Path("test.patch"), tags={"B", "C"}
+                "TEST",
+                "Test-BC",
+                "",
+                path=Path("test.patch"),
+                tags={"B", "C"},
+                feature_tags={"F_B", "F_C"}
             ),
             Patch(
-                "TEST", "Test-BD", "", path=Path("test.patch"), tags={"B", "D"}
+                "TEST",
+                "Test-BD",
+                "",
+                path=Path("test.patch"),
+                tags={"B", "D"},
+                feature_tags={"F_B", "F_D"}
             ),
             Patch(
-                "TEST", "Test-CD", "", path=Path("test.patch"), tags={"C", "D"}
+                "TEST",
+                "Test-CD",
+                "",
+                path=Path("test.patch"),
+                tags={"C", "D"},
+                feature_tags={"F_C", "F_D"}
             ),
             Patch(
                 "TEST",
                 "Test-ABC",
                 "",
                 path=Path("test.patch"),
-                tags={"A", "B", "C"}
+                tags={"A", "B", "C"},
+                feature_tags={"F_A", "F_B", "F_C"}
             ),
             Patch(
                 "TEST",
                 "Test-ABD",
                 "",
                 path=Path("test.patch"),
-                tags={"A", "B", "D"}
+                tags={"A", "B", "D"},
+                feature_tags={"F_A", "F_B", "F_D"}
             ),
             Patch(
                 "TEST",
                 "Test-ACD",
                 "",
                 path=Path("test.patch"),
-                tags={"A", "C", "D"}
+                tags={"A", "C", "D"},
+                feature_tags={"F_A", "F_C", "F_D"}
             ),
             Patch(
                 "TEST",
                 "Test-BCD",
                 "",
                 path=Path("test.patch"),
-                tags={"B", "C", "D"}
+                tags={"B", "C", "D"},
+                feature_tags={"F_B", "F_C", "F_D"}
             ),
         }
 
@@ -311,6 +374,38 @@ def test_any_of_multiple_tags(self):
             for patch in patches:
                 any([tag in patch.tags for tag in tags])
 
+    def test_all_of_single_feature_tag(self):
+        for tag in {"F_A", "F_B", "F_C", "F_D"}:
+            patches = self.patchSet.all_of_features([tag])
+            self.assertEqual(8, len(patches))
+
+    def test_all_of_multiple_feature_tags(self):
+        tags_count = {("F_A", "F_B"): 4,
+                      ("F_C", "F_B"): 4,
+                      ("F_D", "F_B"): 4,
+                      ("F_A", "F_B", "F_C"): 2,
+                      ("F_A", "F_B", "F_C", "F_D"): 1}
+
+        for tags in tags_count:
+            patches = self.patchSet.all_of_features(tags)
+            self.assertEqual(tags_count[tags], len(patches))
+
+    def test_any_of_single_feature_tag(self):
+        for tag in {"F_A", "F_B", "F_C", "F_D"}:
+            patches = self.patchSet.any_of_features([tag])
+            self.assertEqual(8, len(patches))
+
+    def test_any_of_multiple_feature_tags(self):
+        tags_count = {("F_A", "F_B"): 12,
+                      ("F_C", "F_B"): 12,
+                      ("F_D", "F_B"): 12,
+                      ("F_A", "F_B", "F_C"): 14,
+                      ("F_A", "F_B", "F_C", "F_D"): 15}
+
+        for tags in tags_count:
+            patches = self.patchSet.any_of_features(tags)
+            self.assertEqual(tags_count[tags], len(patches))
+
     def test_patchset_intersection(self):
         patches = self.patchSet["A"] & self.patchSet["B"]
 

tests/utils/test_experiment_util.py

@@ -24,6 +24,9 @@
 from varats.project.project_util import BinaryType, ProjectBinaryWrapper
 from varats.project.varats_project import VProject
 from varats.projects.c_projects.xz import Xz
+from varats.projects.perf_tests.feature_perf_cs_collection import (
+    SynthIPTemplate,
+)
 from varats.report.gnu_time_report import TimeReport
 from varats.report.report import FileStatusExtension, ReportSpecification
 from varats.utils.git_util import ShortCommitHash
@@ -419,3 +422,20 @@ def test_get_extra_config_options(self) -> None:
         )
         project = Xz(revision=revision)
         self.assertEqual(EU.get_extra_config_options(project), ["--foo"])
+
+    @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS)
+    def test_get_config_patches(self) -> None:
+        vara_cfg()['paper_config']['current_config'] = "test_config_ids"
+        load_paper_config()
+
+        revision = Revision(
+            SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"),
+            Variant(SynthIPTemplate.SOURCE[1], "4")
+        )
+        project = SynthIPTemplate(revision=revision)
+        patches = EU.get_config_patches(project)
+        self.assertEqual(len(patches), 1)
+        self.assertEqual(
+            list(patches)[0].feature_tags,
+            ["Compress", "fastmode", "smallmode"]
+        )

varats-core/varats/base/configuration.py

@@ -414,3 +414,49 @@ def get_config_value(self, option_name: str) -> tp.Optional[tp.Any]:
 
     def unfreeze(self) -> Configuration:
         return self
+
+
+class PatchConfiguration(Configuration):
+    """Configuration class for projects where configuring is done by applying a
+    patch."""
+
+    def __init__(self, patch_feature_tags: tp.Set[str]):
+        self.__patch_feature_tags: tp.Set[ConfigurationOption] = {
+            ConfigurationOptionImpl(tag, tag) for tag in patch_feature_tags
+        }
+
+    @staticmethod
+    def create_configuration_from_str(config_str: str) -> Configuration:
+        patch_feature_tags = json.loads(config_str)
+        return PatchConfiguration(patch_feature_tags)
+
+    def add_config_option(self, option: ConfigurationOption) -> None:
+        self.__patch_feature_tags.add(option)
+
+    def set_config_option(self, option_name: str, value: tp.Any) -> None:
+        self.__patch_feature_tags = {
+            option for option in self.__patch_feature_tags
+            if option.name != option_name
+        }
+        self.add_config_option(ConfigurationOptionImpl(option_name, value))
+
+    def get_config_value(self, option_name: str) -> tp.Optional[tp.Any]:
+        filtered_options = filter(
+            lambda option: (option.name == option_name),
+            self.__patch_feature_tags
+        )
+        return any(filtered_options)
+
+    def options(self) -> tp.List[ConfigurationOption]:
+        return list(self.__patch_feature_tags)
+
+    def dump_to_string(self) -> str:
+        return ", ".join(
+            map(lambda option: str(option.value), self.__patch_feature_tags)
+        )
+
+    def freeze(self) -> FrozenConfiguration:
+        return FrozenConfiguration(deepcopy(self))
+
+    def unfreeze(self) -> Configuration:
+        return self