Merge branch 'vara-dev' into f-SingleCommitFilter

.github/workflows/ci.yml

@@ -5,6 +5,8 @@ on:
     branches: [ vara, vara-dev ]
   pull_request:
     branches: [ vara, vara-dev ]
+  merge_group:
+    branches: [ vara, vara-dev ]
 
 env:
   BB_TMP_DIR: $(pwd)/benchbuild/tmp

.github/workflows/docs.yml

@@ -3,6 +3,8 @@ name: Docs CI
 on:
   pull_request:
     branches: [ vara, vara-dev ]
+  merge_group:
+    branches: [ vara, vara-dev ]
 
 jobs:
   build:

.github/workflows/pre-commit.yml

@@ -2,6 +2,7 @@ name: pre-commit
 
 on:
   pull_request:
+  merge_group:
 
 jobs:
   pre-commit:

.github/workflows/reviewdog.yml

@@ -3,6 +3,8 @@ name: reviewdog
 on:
   pull_request:
     branches: [ vara, vara-dev ]
+  merge_group:
+    branches: [ vara, vara-dev ]
 
 jobs:
   reviewdog:

.github/workflows/tag.yml

@@ -17,6 +17,11 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.11'
+
       - uses: ./.github/actions/dependenciesActions
       - name: Create and Publish Release
         run: |

docs/source/conf.py

@@ -87,6 +87,10 @@
 # modules that require this module before setting the type checking flag.
 import scipy.stats  # isort:skip
 
+# Matplotlib >=3.8 has a type-checking-flag-guarded import of a symbol that does
+# not exist in the shipped version.
+import matplotlib.pyplot  # isort:skip
+
 # The autodocs typehints plugin does not resolve circular imports caused by type
 # annotations, so we have to manually break the circles.
 import rich.console  # isort:skip

docs/source/tutorials/configuration_specific_experiments.rst

@@ -21,6 +21,7 @@ One just needs to extend the case-study file of a project with a yaml document t
 .. code-block:: yaml
 
   ---
+  config_type: PlainCommandlineConfiguration
   0: '["--foo", "--bar"]'
   1: '["--foo"]'
   ...

docs/source/vara-ts-api/tools/vara-cs-gui.rst

@@ -10,6 +10,7 @@ The gui is started by::
 
 The gui provides 3 Strategies to generate case studies:
     - Manual revision selection: Select revision from the revision history of a project. Multiple revisions can be selected by holding `ctrl` and ranges by holding `shift`. Revisions which are blocked because of bugs in the compilation of the project are marked blue.
+
     .. figure:: vara-cs-gui-manual.png
 
     - Random Sampling: Sample a number of revisions using a random a Normal or HalfNormal Distribution.

requirements.txt

@@ -1,4 +1,4 @@
-benchbuild>=6.6.4
+benchbuild>=6.8
 click>=8.1.3
 distro>=1.5.0
 graphviz>=0.14.2

tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPRuntime_0.case_study

@@ -0,0 +1,25 @@
+---
+DocType: CaseStudy
+Version: 1
+...
+---
+project_name: SynthIPRuntime
+stages:
+- revisions:
+  - commit_hash: 793035062810ea3a2d9a10f831cd199fbbb82090
+    commit_id: 64
+    config_ids:
+    - 0
+    - 1
+    - 2
+    - 3
+    - 4
+version: 0
+...
+---
+config_type: PlainCommandlineConfiguration
+0: '["-d"]'
+1: '["-c"]'
+2: '["-c", "-1"]'
+3: '["-c", "-2"]'
+4: '["-c", "-1", "-2"]'

tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPTemplate_0.case_study

@@ -0,0 +1,25 @@
+---
+DocType: CaseStudy
+Version: 1
+...
+---
+project_name: SynthIPTemplate
+stages:
+- revisions:
+  - commit_hash: 793035062810ea3a2d9a10f831cd199fbbb82090
+    commit_id: 64
+    config_ids:
+    - 0
+    - 1
+    - 2
+    - 3
+    - 4
+version: 0
+...
+---
+config_type: PatchConfiguration
+0: '["Decompress"]'
+1: '["Compress", "no_fastmode", "no_smallmode"]'
+2: '["Compress", "fastmode", "no_smallmode"]'
+3: '["Compress", "no_fastmode", "smallmode"]'
+4: '["Compress", "fastmode", "smallmode"]'

tests/TEST_INPUTS/paper_configs/test_config_ids/SynthSAContextSensitivity_0.case_study

@@ -14,6 +14,7 @@ stages:
 version: 0
 ...
 ---
+config_type: PlainCommandlineConfiguration
 0: '["--compress", "--mem", "10", "8"]'
 1: '["--compress", "--mem", "300", "8"]'
 ...

tests/TEST_INPUTS/paper_configs/test_config_ids/xz_0.case_study

@@ -10,5 +10,6 @@ stages:
     config_ids: [1]
 version: 0
 ---
+config_type: PlainCommandlineConfiguration
 0: '["--foo", "--bar"]'
 1: '["--foo"]'

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/exclude-revision-range.info

@@ -0,0 +1,8 @@
+description: Patch that excludes a range of commits (Otherwise includes all)
+exclude_revisions:
+  revision_range:
+    end: 8ca5cc28e6746eef7340064b5d843631841bf31e
+    start: 01f9f1f07bef22d4248e8349aba4f0c1f204607e
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: exclude-revision-range

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/exclude-single-and-revision-range.info

@@ -0,0 +1,10 @@
+description: Patch that excludes a certain range and individual commits (Otherwise
+  includes all)
+exclude_revisions:
+  revision_range:
+    end: 8ca5cc28e6746eef7340064b5d843631841bf31e
+    start: 01f9f1f07bef22d4248e8349aba4f0c1f204607e
+  single_revision: 27f17080376e409860405c40744887d81d6b3f34
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: exclude-single-and-revision-range

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/exclude-single-revision.info

@@ -0,0 +1,6 @@
+description: Patch that is valid for all commits except a single one
+exclude_revisions:
+  single_revision: 8ca5cc28e6746eef7340064b5d843631841bf31e
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: exclude-single-revision

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/include-range-exclude-range.info

@@ -0,0 +1,12 @@
+description: Patch valid for a range of commits where a subrange is explicitly excluded
+exclude_revisions:
+  revision_range:
+    end: a94fb35ca49719028a1c50bdbc2fb82122043f46
+    start: c051e44a973ee31b3baa571407694467a513ba68
+include_revisions:
+  revision_range:
+    end: 4300ea495e7f013f68e785fdde5c4ead81297999
+    start: 01f9f1f07bef22d4248e8349aba4f0c1f204607e
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: include-range-exclude-range

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/include-range-exclude-single.info

@@ -0,0 +1,10 @@
+description: Patch valid for a range of commits where a single one is excluded
+exclude_revisions:
+  single_revision: 162db88346b06be20faac6976f1ff9bad986accf
+include_revisions:
+  revision_range:
+    end: 8ca5cc28e6746eef7340064b5d843631841bf31e
+    start: 01f9f1f07bef22d4248e8349aba4f0c1f204607e
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: include-range-exclude-single

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/include-revision-range.info

@@ -0,0 +1,8 @@
+description: Patch that is valid for a range of commits
+include_revisions:
+  revision_range:
+    end: 8ca5cc28e6746eef7340064b5d843631841bf31e
+    start: 01f9f1f07bef22d4248e8349aba4f0c1f204607e
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: include-revision-range

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/include-single-and-revision-range.info

@@ -0,0 +1,9 @@
+description: Patch that is valid for a revision range AND another single commit
+include_revisions:
+  revision_range:
+    end: 8ca5cc28e6746eef7340064b5d843631841bf31e
+    start: 01f9f1f07bef22d4248e8349aba4f0c1f204607e
+  single_revision: 27f17080376e409860405c40744887d81d6b3f34
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: include-single-and-revision-range

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/include-single-revision.info

@@ -0,0 +1,6 @@
+description: Patch that is valid for a single revision
+include_revisions:
+  single_revision: 8ca5cc28e6746eef7340064b5d843631841bf31e
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: include-single-revision

tests/TEST_INPUTS/patch_configs/FeaturePerfCSCollection/unrestricted-range.info

@@ -0,0 +1,5 @@
+description: Patch describing an unrestricted range of commits, meaning it should
+  include all commits
+path: bug.patch
+project_name: FeaturePerfCSCollection
+shortname: unrestricted-range

tests/data/test_report.py

@@ -156,6 +156,14 @@ def test_get_uuid(self):
         self.assertEqual(self.report_filename.uuid, self.correct_UUID)
         self.assertRaises(ValueError, lambda: self.broken_report_filename.uuid)
 
+    def test_experiment_shorthand_parsing_with_path_in_name(self) -> None:
+        """Checks that we correctly parse the experiment shorthand also in cases
+        where we have a path as part of the filename."""
+        prefixed = ReportFilename(
+            "/tmp/foobar/" + self.report_filename.filename
+        )
+        self.assertEqual(prefixed.experiment_shorthand, "CRE")
+
 
 class TestConfigReportFilename(unittest.TestCase):
     """Test configuration specific ReportFilename functionality."""

tests/experiment/test_workload_util.py

@@ -6,8 +6,15 @@
 from benchbuild.source.base import Revision, Variant
 
 import varats.experiment.workload_util as wu
+from tests.helper_utils import run_in_test_environment, UnitTestFixtures
+from varats.paper.paper_config import load_paper_config
 from varats.projects.c_projects.xz import Xz
+from varats.projects.perf_tests.feature_perf_cs_collection import (
+    SynthIPTemplate,
+    SynthIPRuntime,
+)
 from varats.utils.git_util import ShortCommitHash
+from varats.utils.settings import vara_cfg
 
 TT = PathToken.make_token(RootRenderer())
 
@@ -52,6 +59,73 @@ def test_workload_commands_tags_selected(self) -> None:
         )
         self.assertEqual(len(commands), 1)
 
+    def test_workload_commands_requires(self) -> None:
+        revision = Revision(Xz, Variant(Xz.SOURCE[0], "c5c7ceb08a"))
+        project = Xz(revision=revision)
+        binary = Xz.binaries_for_revision(ShortCommitHash("c5c7ceb08a"))[0]
+
+        commands = wu.workload_commands(
+            project, binary, [wu.WorkloadCategory.EXAMPLE]
+        )
+        self.assertEqual(len(commands), 1)
+        commands = wu.workload_commands(
+            project, binary, [wu.WorkloadCategory.MEDIUM]
+        )
+        self.assertEqual(len(commands), 1)
+
+    @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS)
+    def test_workload_config_param_token(self) -> None:
+        vara_cfg()['paper_config']['current_config'] = "test_config_ids"
+        load_paper_config()
+
+        revision = Revision(
+            SynthIPRuntime, Variant(SynthIPRuntime.SOURCE[0], "7930350628"),
+            Variant(SynthIPRuntime.SOURCE[1], "1")
+        )
+        project = SynthIPRuntime(revision=revision)
+        binary = SynthIPRuntime.binaries_for_revision(
+            ShortCommitHash("7930350628")
+        )[0]
+
+        commands = wu.workload_commands(
+            project, binary, [wu.WorkloadCategory.SMALL]
+        )
+        self.assertEqual(len(commands), 1)
+        command = commands[0]
+        args = command.command.rendered_args(project=project)
+        self.assertEquals(args, tuple(["-c"]))
+
+    @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS)
+    def test_workload_commands_requires_patch(self) -> None:
+        vara_cfg()['paper_config']['current_config'] = "test_config_ids"
+        load_paper_config()
+
+        revision = Revision(
+            SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"),
+            Variant(SynthIPTemplate.SOURCE[1], "1")
+        )
+        project = SynthIPTemplate(revision=revision)
+        binary = SynthIPTemplate.binaries_for_revision(
+            ShortCommitHash("7930350628")
+        )[0]
+        workloads = wu.workload_commands(project, binary, [])
+        self.assertEqual(2, len(workloads))
+
+    @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS)
+    def test_workload_commands_requires_patch2(self) -> None:
+        vara_cfg()['paper_config']['current_config'] = "test_config_ids"
+        load_paper_config()
+
+        revision = Revision(
+            SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"),
+            Variant(SynthIPTemplate.SOURCE[1], "0")
+        )
+        project = SynthIPTemplate(revision=revision)
+        binary = SynthIPTemplate \
+            .binaries_for_revision(ShortCommitHash("7930350628"))[0]
+        workloads = wu.workload_commands(project, binary, [])
+        self.assertEqual(0, len(workloads))
+
 
 class TestWorkloadFilenames(unittest.TestCase):
 

tests/paper/test_case_study.py

@@ -48,6 +48,7 @@
     commit_id: 494
 ...
 ---
+config_type: ConfigurationImpl
 0: '{"foo": true, "bar": false, "bazz": "bazz-value", "buzz": "None"}'
 1: '{}'
 2: '{}'