Implements feature based perf instrumentation comparision (#806)

Introduces a new setup for measuring feature performance

requirements.txt

@@ -1,5 +1,6 @@
 benchbuild>=6.8
 click>=8.1.3
+cliffs-delta>=1.0.0
 distro>=1.5.0
 graphviz>=0.14.2
 ijson>=3.1.4

varats-core/varats/experiment/experiment_util.py

@@ -518,9 +518,12 @@ def call_with_output_folder(self, tmp_dir: Path) -> StepResult:
         """Actual call implementation that gets a path to tmp_folder."""
 
 
-class ZippedExperimentSteps(
-    MultiStep[tp.Union[OutputFolderStep, ProjectStep]]  # type: ignore
-):
+ZippedStepTy = tp.TypeVar(
+    "ZippedStepTy", bound=tp.Union[OutputFolderStep, ProjectStep]
+)
+
+
+class ZippedExperimentSteps(MultiStep[ZippedStepTy]):  # type: ignore
     """Runs multiple actions, providing them a shared tmp folder that afterwards
     is zipped into an archive."""
 
@@ -529,7 +532,7 @@ class ZippedExperimentSteps(
 
     def __init__(
         self, output_filepath: ReportFilepath,
-        actions: tp.Optional[tp.List[tp.Union[OutputFolderStep, ProjectStep]]]
+        actions: tp.Optional[tp.List[ZippedStepTy]]
     ) -> None:
         super().__init__(actions)
         self.__output_filepath = output_filepath

varats-core/varats/experiment/steps/patch.py

@@ -48,7 +48,7 @@ class RevertPatch(actions.ProjectStep):
     NAME = "REVERT_PATCH"
     DESCRIPTION = "Revert a Git patch from a project."
 
-    def __init__(self, project, patch):
+    def __init__(self, project: VProject, patch: Patch) -> None:
         super().__init__(project)
         self.__patch = patch
 

varats-core/varats/project/varats_command.py

@@ -1,7 +1,12 @@
 """Custom version of benchbuild's Command for use with the VaRA-Tool-Suite."""
 import typing as tp
+from pathlib import Path
 
 from benchbuild.command import Command, ProjectCommand, PathToken
+from benchbuild.utils.cmd import time
+from plumbum import local
+from plumbum.commands.base import BaseCommand
+from plumbum.machines import LocalCommand
 
 from varats.utils.config import get_config_patches
 
@@ -75,6 +80,39 @@ def as_plumbum(self, **kwargs: tp.Any) -> 'BoundEnvCommand':
 
         return cmd
 
+    def as_plumbum_wrapped_with(
+        self,
+        wrapper_cmd: tp.Optional['BoundEnvCommand'] = None,
+        adapted_binary_location: tp.Optional[Path] = None,
+        **kwargs: tp.Any
+    ) -> 'BaseCommand':
+        base_cmd = super().as_plumbum(**kwargs)
+
+        # TODO: maybe we should just provide a callable to modify the original
+        # command
+        if adapted_binary_location:
+            if isinstance(base_cmd, LocalCommand):
+                base_cmd.executable = base_cmd.executable.copy(
+                    adapted_binary_location, override=True
+                )
+            else:
+                base_cmd.cmd.executable = base_cmd.cmd.executable.copy(
+                    adapted_binary_location, override=True
+                )
+
+        if wrapper_cmd:
+            cmd = wrapper_cmd[base_cmd]
+        else:
+            cmd = base_cmd
+
+        if self._redirect_stdin:
+            cmd = cmd < str(self._redirect_stdin.render(**kwargs))
+
+        if self._redirect_stdout:
+            cmd = cmd > str(self._redirect_stdout.render(**kwargs))
+
+        return cmd
+
 
 class VProjectCommand(ProjectCommand):  # type: ignore
 

varats/setup.py

@@ -30,7 +30,7 @@
         "pandas>=1.5.3",
         "plotly>=5.13.1",
         "plumbum>=1.6",
-        "pygit2>=1.10",
+        "pygit2>=1.10,<1.14.0",
         "PyGithub>=1.47",
         "pygraphviz>=1.7",
         "pygtrie>=2.3",
@@ -44,6 +44,7 @@
         "tabulate>=0.9",
         "varats-core>=13.0.5",
         "wllvm>=1.3.1",
+        "cliffs-delta>=1.0.0",
     ],
     author="Florian Sattler",
     author_email="[email protected]",