Merge branch 'vara-dev' into f-DuneExamples

docs/source/conf.py

@@ -87,6 +87,10 @@
 # modules that require this module before setting the type checking flag.
 import scipy.stats  # isort:skip
 
+# Matplotlib >=3.8 has a type-checking-flag-guarded import of a symbol that does
+# not exist in the shipped version.
+import matplotlib.pyplot  # isort:skip
+
 # The autodocs typehints plugin does not resolve circular imports caused by type
 # annotations, so we have to manually break the circles.
 import rich.console  # isort:skip

docs/source/tutorials/configuration_specific_experiments.rst

@@ -21,6 +21,7 @@ One just needs to extend the case-study file of a project with a yaml document t
 .. code-block:: yaml
 
   ---
+  config_type: PlainCommandlineConfiguration
   0: '["--foo", "--bar"]'
   1: '["--foo"]'
   ...

docs/source/vara-ts-api/tools/vara-cs-gui.rst

@@ -10,6 +10,7 @@ The gui is started by::
 
 The gui provides 3 Strategies to generate case studies:
     - Manual revision selection: Select revision from the revision history of a project. Multiple revisions can be selected by holding `ctrl` and ranges by holding `shift`. Revisions which are blocked because of bugs in the compilation of the project are marked blue.
+
     .. figure:: vara-cs-gui-manual.png
 
     - Random Sampling: Sample a number of revisions using a random a Normal or HalfNormal Distribution.

tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPTemplate_0.case_study

@@ -0,0 +1,25 @@
+---
+DocType: CaseStudy
+Version: 1
+...
+---
+project_name: SynthIPTemplate
+stages:
+- revisions:
+  - commit_hash: 793035062810ea3a2d9a10f831cd199fbbb82090
+    commit_id: 64
+    config_ids:
+    - 0
+    - 1
+    - 2
+    - 3
+    - 4
+version: 0
+...
+---
+config_type: PatchConfiguration
+0: '["Decompress"]'
+1: '["Compress"]'
+2: '["Compress", "fastmode", "no_smallmode"]'
+3: '["Compress", "no_fastmode", "smallmode"]'
+4: '["Compress", "fastmode", "smallmode"]'

tests/TEST_INPUTS/paper_configs/test_config_ids/SynthSAContextSensitivity_0.case_study

@@ -14,6 +14,7 @@ stages:
 version: 0
 ...
 ---
+config_type: PlainCommandlineConfiguration
 0: '["--compress", "--mem", "10", "8"]'
 1: '["--compress", "--mem", "300", "8"]'
 ...

tests/TEST_INPUTS/paper_configs/test_config_ids/xz_0.case_study

@@ -10,5 +10,6 @@ stages:
     config_ids: [1]
 version: 0
 ---
+config_type: PlainCommandlineConfiguration
 0: '["--foo", "--bar"]'
 1: '["--foo"]'

tests/data/test_report.py

@@ -156,6 +156,14 @@ def test_get_uuid(self):
         self.assertEqual(self.report_filename.uuid, self.correct_UUID)
         self.assertRaises(ValueError, lambda: self.broken_report_filename.uuid)
 
+    def test_experiment_shorthand_parsing_with_path_in_name(self) -> None:
+        """Checks that we correctly parse the experiment shorthand also in cases
+        where we have a path as part of the filename."""
+        prefixed = ReportFilename(
+            "/tmp/foobar/" + self.report_filename.filename
+        )
+        self.assertEqual(prefixed.experiment_shorthand, "CRE")
+
 
 class TestConfigReportFilename(unittest.TestCase):
     """Test configuration specific ReportFilename functionality."""

tests/experiment/test_workload_util.py

@@ -6,8 +6,14 @@
 from benchbuild.source.base import Revision, Variant
 
 import varats.experiment.workload_util as wu
+from tests.helper_utils import run_in_test_environment, UnitTestFixtures
+from varats.paper.paper_config import load_paper_config
 from varats.projects.c_projects.xz import Xz
+from varats.projects.perf_tests.feature_perf_cs_collection import (
+    SynthIPTemplate,
+)
 from varats.utils.git_util import ShortCommitHash
+from varats.utils.settings import vara_cfg
 
 TT = PathToken.make_token(RootRenderer())
 
@@ -52,6 +58,51 @@ def test_workload_commands_tags_selected(self) -> None:
         )
         self.assertEqual(len(commands), 1)
 
+    def test_workload_commands_requires(self) -> None:
+        revision = Revision(Xz, Variant(Xz.SOURCE[0], "c5c7ceb08a"))
+        project = Xz(revision=revision)
+        binary = Xz.binaries_for_revision(ShortCommitHash("c5c7ceb08a"))[0]
+
+        commands = wu.workload_commands(
+            project, binary, [wu.WorkloadCategory.EXAMPLE]
+        )
+        self.assertEqual(len(commands), 1)
+        commands = wu.workload_commands(
+            project, binary, [wu.WorkloadCategory.MEDIUM]
+        )
+        self.assertEqual(len(commands), 1)
+
+    @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS)
+    def test_workload_commands_requires_patch(self) -> None:
+        vara_cfg()['paper_config']['current_config'] = "test_config_ids"
+        load_paper_config()
+
+        revision = Revision(
+            SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"),
+            Variant(SynthIPTemplate.SOURCE[1], "1")
+        )
+        project = SynthIPTemplate(revision=revision)
+        binary = SynthIPTemplate.binaries_for_revision(
+            ShortCommitHash("7930350628")
+        )[0]
+        workloads = wu.workload_commands(project, binary, [])
+        self.assertEqual(len(workloads), 2)
+
+    @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS)
+    def test_workload_commands_requires_patch2(self) -> None:
+        vara_cfg()['paper_config']['current_config'] = "test_config_ids"
+        load_paper_config()
+
+        revision = Revision(
+            SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"),
+            Variant(SynthIPTemplate.SOURCE[1], "0")
+        )
+        project = SynthIPTemplate(revision=revision)
+        binary = SynthIPTemplate \
+            .binaries_for_revision(ShortCommitHash("7930350628"))[0]
+        workloads = wu.workload_commands(project, binary, [])
+        self.assertEqual(len(workloads), 0)
+
 
 class TestWorkloadFilenames(unittest.TestCase):
 

tests/paper/test_case_study.py

@@ -48,6 +48,7 @@
     commit_id: 494
 ...
 ---
+config_type: ConfigurationImpl
 0: '{"foo": true, "bar": false, "bazz": "bazz-value", "buzz": "None"}'
 1: '{}'
 2: '{}'

tests/paper_mgmt/test_case_study.py

@@ -239,7 +239,8 @@ def test_get_newest_result_files_for_case_study_with_empty_res_dir(
         UnitTestFixtures.PAPER_CONFIGS, UnitTestFixtures.RESULT_FILES
     )
     def test_get_newest_result_files_for_case_study_with_config(self) -> None:
-        """Check that when we have two files, the newes one get's selected."""
+        """Check that when we have two files that differ in their config id,
+        both get selected."""
         vara_cfg()['paper_config']['current_config'] = "test_config_ids"
         load_paper_config()
 
@@ -273,7 +274,56 @@ def test_get_newest_result_files_for_case_study_with_config(self) -> None:
 
         self.assertEqual(newest_res_filenames[0].config_id, 0)
         self.assertEqual(newest_res_filenames[1].config_id, 1)
-        self.assertEqual(len(newest_res_filenames), 2)
+        self.assertEqual(newest_res_filenames[2].config_id, 0)
+        self.assertEqual(newest_res_filenames[3].config_id, 1)
+        self.assertEqual(len(newest_res_filenames), 4)
+
+    @run_in_test_environment(
+        UnitTestFixtures.PAPER_CONFIGS, UnitTestFixtures.RESULT_FILES
+    )
+    def test_get_newest_result_files_for_case_study_with_diff_exp(self) -> None:
+        """Check that when we have two files that differ in their experiment
+        shorthand, both get selected."""
+        vara_cfg()['paper_config']['current_config'] = "test_config_ids"
+        load_paper_config()
+
+        config_0_file = ReportFilename(
+            "BBBase-CR-SynthSAContextSensitivity-ContextSense-06eac0edb6/"
+            "b24ee2c1-fc85-47ba-abbd-90c98e88a37c_config-0_success.zip"
+        )
+        config_1_file = ReportFilename(
+            "BBBaseO-CR-SynthSAContextSensitivity-ContextSense-06eac0edb6/"
+            "b24ee2c1-fc85-47ba-abbd-90c98e88a37c_config-0_success.zip"
+        )
+
+        now = datetime.now().timestamp()
+        file_path_0 = Path(
+            str(vara_cfg()['result_dir'])
+        ) / 'SynthSAContextSensitivity' / config_0_file.filename
+        os.utime(file_path_0, (now, now))
+
+        file_path_1 = Path(
+            str(vara_cfg()['result_dir'])
+        ) / 'SynthSAContextSensitivity' / config_1_file.filename
+        os.utime(file_path_1, (now, now))
+
+        newest_res_files = MCS.get_newest_result_files_for_case_study(
+            get_paper_config().get_case_studies('SynthSAContextSensitivity')[0],
+            Path(vara_cfg()['result_dir'].value), CR
+        )
+
+        newest_res_files.sort(reverse=True)
+        newest_res_filenames = [ReportFilename(x) for x in newest_res_files]
+
+        self.assertEqual(
+            newest_res_filenames[0].experiment_shorthand, "BBBaseO"
+        )
+        self.assertEqual(
+            newest_res_filenames[1].experiment_shorthand, "BBBaseO"
+        )
+        self.assertEqual(newest_res_filenames[2].experiment_shorthand, "BBBase")
+        self.assertEqual(newest_res_filenames[3].experiment_shorthand, "BBBase")
+        self.assertEqual(len(newest_res_filenames), 4)
 
     def test_get_case_study_file_name_filter_empty(self) -> None:
         """Check that we correctly handle  case study filter generation even if