From 3c019dbc0ca5c199c90ef69b7a99c9e7920e886d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20B=C3=B6hm?= Date: Thu, 7 Sep 2023 16:18:29 +0200 Subject: [PATCH 1/8] Always load BB config from environment. (#833) Our load order for the BB config caused the "varats" block to always use default values if running benchbuild directly. This had the effect that container runs would write BC files to an incorrect directory that wasn't mounted from the outside. Therefore, BC files would never be cached when using containers. --- varats-core/varats/utils/settings.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/varats-core/varats/utils/settings.py b/varats-core/varats/utils/settings.py index bac6860ad..e8b207cf9 100644 --- a/varats-core/varats/utils/settings.py +++ b/varats-core/varats/utils/settings.py @@ -283,7 +283,10 @@ def bb_cfg() -> s.Configuration: bb_cfg_path = Path(bb_root) / ".benchbuild.yml" if bb_cfg_path.exists(): BB_CFG.load(local.path(bb_cfg_path)) - BB_CFG.init_from_env() + + # Environment should always override config files + BB_CFG.init_from_env() + _BB_CFG = BB_CFG create_missing_bb_folders() return _BB_CFG From 04bfd569c9c9fbff8e0041a365204ce5b2904625 Mon Sep 17 00:00:00 2001 From: danjujan <44864658+danjujan@users.noreply.github.com> Date: Tue, 12 Sep 2023 22:31:35 +0200 Subject: [PATCH 2/8] Adds requires attribute to Command (#822) The requires attribute is used by workload_commands to check if the command includes all necessary arguments. --- tests/experiment/test_workload_util.py | 14 ++++++++ .../varats/experiment/workload_util.py | 28 +++++++++++++++- varats-core/varats/project/project_util.py | 33 +++++++++++++++++++ varats/varats/projects/c_projects/xz.py | 20 +++++++---- 4 files changed, 88 insertions(+), 7 deletions(-) diff --git a/tests/experiment/test_workload_util.py b/tests/experiment/test_workload_util.py index 729dedc24..b8de4db27 100644 --- a/tests/experiment/test_workload_util.py +++ b/tests/experiment/test_workload_util.py @@ -52,6 +52,20 @@ def test_workload_commands_tags_selected(self) -> None: ) self.assertEqual(len(commands), 1) + def test_workload_commands_requires(self) -> None: + revision = Revision(Xz, Variant(Xz.SOURCE[0], "c5c7ceb08a")) + project = Xz(revision=revision) + binary = Xz.binaries_for_revision(ShortCommitHash("c5c7ceb08a"))[0] + + commands = wu.workload_commands( + project, binary, [wu.WorkloadCategory.EXAMPLE] + ) + self.assertEqual(len(commands), 1) + commands = wu.workload_commands( + project, binary, [wu.WorkloadCategory.MEDIUM] + ) + self.assertEqual(len(commands), 1) + class TestWorkloadFilenames(unittest.TestCase): diff --git a/varats-core/varats/experiment/workload_util.py b/varats-core/varats/experiment/workload_util.py index 8cf66daff..4566e2ee7 100644 --- a/varats-core/varats/experiment/workload_util.py +++ b/varats-core/varats/experiment/workload_util.py @@ -19,6 +19,7 @@ Command, ) +from varats.experiment.experiment_util import get_extra_config_options from varats.project.project_util import ProjectBinaryWrapper from varats.project.varats_project import VProject from varats.report.report import KeyedReportAggregate, ReportTy @@ -92,8 +93,33 @@ def workload_commands( ) ] + # Filter commands that have required args set. + extra_options = set(get_extra_config_options(project)) + + def requires_any_filter(prj_cmd: ProjectCommand) -> bool: + if hasattr( + prj_cmd.command, "requires_any" + ) and prj_cmd.command.requires_any: + args = set(prj_cmd.command._args).union(extra_options) + return bool(args.intersection(prj_cmd.command.requires_any)) + return True + + def requires_all_filter(prj_cmd: ProjectCommand) -> bool: + if hasattr( + prj_cmd.command, "requires_all" + ) and prj_cmd.command.requires_all: + args = set(prj_cmd.command._args).union(extra_options) + return bool(prj_cmd.command.requires_all.issubset(args)) + return True + + available_cmds = filter( + requires_all_filter, filter(requires_any_filter, project_cmds) + ) + return list( - filter(lambda prj_cmd: prj_cmd.path.name == binary.name, project_cmds) + filter( + lambda prj_cmd: prj_cmd.path.name == binary.name, available_cmds + ) ) diff --git a/varats-core/varats/project/project_util.py b/varats-core/varats/project/project_util.py index a4c27d74d..3028c438b 100644 --- a/varats-core/varats/project/project_util.py +++ b/varats-core/varats/project/project_util.py @@ -7,6 +7,7 @@ import benchbuild as bb import pygit2 +from benchbuild.command import Command from benchbuild.source import Git from benchbuild.utils.cmd import git from plumbum import local @@ -382,3 +383,35 @@ def copy_renamed_git_to_dest(src_dir: Path, dest_dir: Path) -> None: for name in dirs: if name == ".gitted": os.rename(os.path.join(root, name), os.path.join(root, ".git")) + + +class VCommand(Command): # type: ignore [misc] + """ + Wrapper around benchbuild's Command class. + + Attributes: + requires_any: sufficient args that must be available for successful execution. + requires_all: all args that must be available for successful execution. + """ + + _requires: tp.Set[str] + + def __init__( + self, + *args: tp.Any, + requires_any: tp.Optional[tp.Set[str]] = None, + requires_all: tp.Optional[tp.Set[str]] = None, + **kwargs: tp.Union[str, tp.List[str]], + ) -> None: + + super().__init__(*args, **kwargs) + self._requires_any = requires_any if requires_any else set() + self._requires_all = requires_all if requires_all else set() + + @property + def requires_any(self) -> tp.Set[str]: + return self._requires_any + + @property + def requires_all(self) -> tp.Set[str]: + return self._requires_all diff --git a/varats/varats/projects/c_projects/xz.py b/varats/varats/projects/c_projects/xz.py index 1fac7c349..3299a0e08 100644 --- a/varats/varats/projects/c_projects/xz.py +++ b/varats/varats/projects/c_projects/xz.py @@ -2,7 +2,7 @@ import typing as tp import benchbuild as bb -from benchbuild.command import Command, SourceRoot, WorkloadSet +from benchbuild.command import SourceRoot, WorkloadSet from benchbuild.source import HTTPMultiple from benchbuild.utils.cmd import autoreconf, make from benchbuild.utils.revision_ranges import ( @@ -18,6 +18,7 @@ from varats.paper.paper_config import PaperConfigSpecificGit from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( + VCommand, ProjectBinaryWrapper, get_local_project_git_path, BinaryType, @@ -84,16 +85,19 @@ class Xz(VProject): WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ - Command( + VCommand( SourceRoot("xz") / RSBinary("xz"), "-k", - "geo-maps/countries-land-1km.geo.json", + # Use output_param to ensure input file + # gets appended after all arguments. + output_param=["{output}"], + output=SourceRoot("geo-maps/countries-land-250m.geo.json"), label="countries-land-1km", creates=["geo-maps/countries-land-1km.geo.json.xz"] ) ], WorkloadSet(WorkloadCategory.MEDIUM): [ - Command( + VCommand( SourceRoot("xz") / RSBinary("xz"), "-k", "-9e", @@ -101,9 +105,13 @@ class Xz(VProject): "--threads=1", "--format=xz", "-vv", - "geo-maps/countries-land-250m.geo.json", + # Use output_param to ensure input file + # gets appended after all arguments. + output_param=["{output}"], + output=SourceRoot("geo-maps/countries-land-250m.geo.json"), label="countries-land-250m", - creates=["geo-maps/countries-land-250m.geo.json.xz"] + creates=["geo-maps/countries-land-250m.geo.json.xz"], + requires_all={"--compress"}, ) ], } From 3d6b4ffa794d9cdf1142ab891730eef6eba97b77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20B=C3=B6hm?= Date: Fri, 22 Sep 2023 08:06:40 +0200 Subject: [PATCH 3/8] Fixes docs break with matplotlib >=3.8 (#837) --- docs/source/conf.py | 4 ++++ docs/source/vara-ts-api/tools/vara-cs-gui.rst | 1 + varats/varats/data/metrics.py | 12 +++++++----- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index ef57a97cf..54392b0f2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -87,6 +87,10 @@ # modules that require this module before setting the type checking flag. import scipy.stats # isort:skip +# Matplotlib >=3.8 has a type-checking-flag-guarded import of a symbol that does +# not exist in the shipped version. +import matplotlib.pyplot # isort:skip + # The autodocs typehints plugin does not resolve circular imports caused by type # annotations, so we have to manually break the circles. import rich.console # isort:skip diff --git a/docs/source/vara-ts-api/tools/vara-cs-gui.rst b/docs/source/vara-ts-api/tools/vara-cs-gui.rst index 96c3493a7..dcdd6219c 100644 --- a/docs/source/vara-ts-api/tools/vara-cs-gui.rst +++ b/docs/source/vara-ts-api/tools/vara-cs-gui.rst @@ -10,6 +10,7 @@ The gui is started by:: The gui provides 3 Strategies to generate case studies: - Manual revision selection: Select revision from the revision history of a project. Multiple revisions can be selected by holding `ctrl` and ranges by holding `shift`. Revisions which are blocked because of bugs in the compilation of the project are marked blue. + .. figure:: vara-cs-gui-manual.png - Random Sampling: Sample a number of revisions using a random a Normal or HalfNormal Distribution. diff --git a/varats/varats/data/metrics.py b/varats/varats/data/metrics.py index 78102735e..4524dc205 100644 --- a/varats/varats/data/metrics.py +++ b/varats/varats/data/metrics.py @@ -28,7 +28,7 @@ def gini_coefficient(distribution: pd.Series) -> float: Calculates the Gini coefficient of the data. For more information see online - `gini coefficient `_. + `Gini coefficient `_. Args: distribution: sorted series to calculate the Gini coefficient for @@ -141,10 +141,12 @@ class ConfusionMatrix(tp.Generic[T]): """ Helper class to automatically calculate classification results. - | Predicted Positive (PP) | Predicted Negative (PN) - --------------------|---------------------------|-------------------------- - Actual Positive (P) | True Positive (TP) | False Negative (FN) - Actual Negative (N) | False Positive (FP) | True Negative (TN) + +---------------------+-------------------------+-------------------------+ + | | Predicted Positive (PP) | Predicted Negative (PN) | + +---------------------+-------------------------+-------------------------+ + | Actual Positive (P) | True Positive (TP) | False Negative (FN) | + | Actual Negative (N) | False Positive (FP) | True Negative (TN) | + +---------------------+-------------------------+-------------------------+ Reference: https://en.wikipedia.org/wiki/Precision_and_recall """ From b344eb428c12803502515143b3459b76dfc243dd Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Fri, 22 Sep 2023 08:27:43 +0200 Subject: [PATCH 4/8] Adds support for SingleRevision for the only_valid_in parameter of RevisionBinaryMap (#836) resolves se-sic/VaRA#1040 --- tests/utils/test_git_util.py | 13 ++++++++++++- varats-core/varats/utils/git_util.py | 10 +++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/tests/utils/test_git_util.py b/tests/utils/test_git_util.py index 1450add2a..c11818665 100644 --- a/tests/utils/test_git_util.py +++ b/tests/utils/test_git_util.py @@ -2,7 +2,7 @@ import unittest from pathlib import Path -from benchbuild.utils.revision_ranges import RevisionRange +from benchbuild.utils.revision_ranges import RevisionRange, SingleRevision from varats.project.project_util import ( get_local_project_git, @@ -568,6 +568,17 @@ def test_specification_validity_range_multiple_binaries(self) -> None: self.assertIn("SingleLocalMultipleRegions", self.rv_map) self.assertIn("SingleLocalSimple", self.rv_map) + def test_specification_single_revision(self) -> None: + """Check if we can add binaries that are only valid with a single + revision.""" + self.rv_map.specify_binary( + "build/bin/SingleLocalMultipleRegions", + BinaryType.EXECUTABLE, + only_valid_in=SingleRevision("162db88346") + ) + + self.assertIn("SingleLocalMultipleRegions", self.rv_map) + def test_specification_binaries_with_special_name(self) -> None: """Check if we can add binaries that have a special name.""" self.rv_map.specify_binary( diff --git a/varats-core/varats/utils/git_util.py b/varats-core/varats/utils/git_util.py index 413ecd269..6f0cd1c1d 100644 --- a/varats-core/varats/utils/git_util.py +++ b/varats-core/varats/utils/git_util.py @@ -1063,7 +1063,9 @@ def specify_binary( override_entry_point = kwargs.get("override_entry_point", None) if override_entry_point: override_entry_point = Path(override_entry_point) - validity_range = kwargs.get("only_valid_in", None) + validity_range: AbstractRevisionRange = kwargs.get( + "only_valid_in", None + ) valid_exit_codes = kwargs.get("valid_exit_codes", None) wrapped_binary = ProjectBinaryWrapper( @@ -1072,6 +1074,7 @@ def specify_binary( ) if validity_range: + validity_range.init_cache(self.__repo_location) self.__revision_specific_mappings[validity_range].append( wrapped_binary ) @@ -1087,10 +1090,7 @@ def __getitem__(self, for validity_range, wrapped_binaries \ in self.__revision_specific_mappings.items(): - if revision in get_all_revisions_between( - validity_range.id_start, validity_range.id_end, ShortCommitHash, - self.__repo_location - ): + if revision in map(ShortCommitHash, validity_range): revision_specific_binaries.extend(wrapped_binaries) revision_specific_binaries.extend(self.__always_valid_mappings) From af451e3a9dc67339e35783b463e5a5ec8d78a4a8 Mon Sep 17 00:00:00 2001 From: Lukas Abelt Date: Fri, 22 Sep 2023 09:47:09 +0200 Subject: [PATCH 5/8] Fixes Patch Provider parallel patch repo update (#835) Adds some safeguards to avoid pulling concurrently when running experiments on the cluster. resolves se-sic/VaRA#1038 --- .../varats/provider/patch/patch_provider.py | 19 ++++++++++++------- varats-core/varats/utils/filesystem_util.py | 16 +++++++++++++++- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/varats-core/varats/provider/patch/patch_provider.py b/varats-core/varats/provider/patch/patch_provider.py index b370df208..d466a629b 100644 --- a/varats-core/varats/provider/patch/patch_provider.py +++ b/varats-core/varats/provider/patch/patch_provider.py @@ -18,6 +18,7 @@ from varats.project.project_util import get_local_project_git_path from varats.provider.provider import Provider, ProviderType +from varats.utils.filesystem_util import lock_file from varats.utils.git_commands import pull_current_branch, fetch_repository from varats.utils.git_util import ( CommitHash, @@ -237,12 +238,10 @@ class PatchProvider(Provider): def __init__(self, project: tp.Type[Project]): super().__init__(project) - # BB only performs a fetch so our repo might be out of date - pull_current_branch(self._get_patches_repository_path()) + self._update_local_patches_repo() + repo_path = self._get_patches_repository_path() - patches_project_dir = Path( - self._get_patches_repository_path() / self.project.NAME - ) + patches_project_dir = repo_path / self.project.NAME if not patches_project_dir.is_dir(): warnings.warn( @@ -316,6 +315,12 @@ def create_default_provider( @classmethod def _get_patches_repository_path(cls) -> Path: - cls.patches_source.fetch() - return Path(target_prefix()) / cls.patches_source.local + + @classmethod + def _update_local_patches_repo(cls): + lock_path = Path(target_prefix()) / "patch_provider.lock" + + with lock_file(lock_path): + cls.patches_source.fetch() + pull_current_branch(cls._get_patches_repository_path()) diff --git a/varats-core/varats/utils/filesystem_util.py b/varats-core/varats/utils/filesystem_util.py index 6f71f01d9..bc44c3265 100644 --- a/varats-core/varats/utils/filesystem_util.py +++ b/varats-core/varats/utils/filesystem_util.py @@ -1,6 +1,8 @@ """Utility functions for handling filesystem related tasks.""" - +import fcntl +import os.path import typing as tp +from contextlib import contextmanager from pathlib import Path @@ -13,3 +15,15 @@ def __init__(self, folder: tp.Union[Path, str]) -> None: f"Folder: '{str(folder)}' should be created " "but was already present." ) + + +@contextmanager +def lock_file(lock_path: Path, lock_mode: int = fcntl.LOCK_EX) -> tp.Generator: + open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC + lock_fd = os.open(lock_path, open_mode) + try: + fcntl.flock(lock_fd, lock_mode) + yield + finally: + fcntl.flock(lock_fd, fcntl.LOCK_UN) + os.close(lock_fd) From 4b0e29b9d6e5f97f4cf7f883715f22e0d18e8ea1 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Sat, 23 Sep 2023 16:07:18 +0200 Subject: [PATCH 6/8] Fixes packaging bug with multiple exp that have the same report type (#832) --- ...4c6-8ce0-08d0a29c677b_config-1_success.zip | Bin 0 -> 4970 bytes ...7ba-abbd-90c98e88a37c_config-0_success.zip | Bin 0 -> 5145 bytes tests/data/test_report.py | 8 +++ tests/paper_mgmt/test_case_study.py | 54 +++++++++++++++++- varats-core/varats/report/report.py | 2 +- varats/varats/paper_mgmt/case_study.py | 17 ++++-- 6 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 tests/TEST_INPUTS/results/SynthSAContextSensitivity/BBBaseO-CR-SynthSAContextSensitivity-ContextSense-06eac0edb6/8380144f-9a25-44c6-8ce0-08d0a29c677b_config-1_success.zip create mode 100644 tests/TEST_INPUTS/results/SynthSAContextSensitivity/BBBaseO-CR-SynthSAContextSensitivity-ContextSense-06eac0edb6/b24ee2c1-fc85-47ba-abbd-90c98e88a37c_config-0_success.zip diff --git a/tests/TEST_INPUTS/results/SynthSAContextSensitivity/BBBaseO-CR-SynthSAContextSensitivity-ContextSense-06eac0edb6/8380144f-9a25-44c6-8ce0-08d0a29c677b_config-1_success.zip b/tests/TEST_INPUTS/results/SynthSAContextSensitivity/BBBaseO-CR-SynthSAContextSensitivity-ContextSense-06eac0edb6/8380144f-9a25-44c6-8ce0-08d0a29c677b_config-1_success.zip new file mode 100644 index 0000000000000000000000000000000000000000..df6194dc1212a7b0d6ff924bace3515c4cd713d2 GIT binary patch literal 4970 zcmbW5WmFX2w!miq0SQ4$LO_tATM+3G6&R$uYv}G!N{~_zkRC#MNa>CNhVIT8K)So* za_@WZ-v5XDV!d-doPEw&Ywz>v{MK(PfzdHY0000s;P&%{M)skv^VlN*fE)_|5PUdw zGoD>>zfwR(4jd4>yj6b`WD1H)n_)#NO3~)7#21 zZo=!$xYzU6Lt{MWphSv~)n&0R)lNT6IqKYMDLnX+R-j#*fGM&%5?7ebd*^FSkz{#m_X1fE0w-40t+Gi*@52 z*iqw00TOE5*x7*j<)$H`o6oWsP>LjJx*(O`7^B)~U!^-QV$cFvYCl{5RFad<#P@2| z;;sKY%Kv1wKV5UEmQFn45%KY=R{lv1O_VqMoaQA9-JR8hI$}g;(A-e9xJ&*?SzU_GScp+kD+}Uwl1%3hW$l7< zz26wC)N)%CWO%zF=Lh}=#8I@!GT_=2o8|Jo_da8%*sjAkgACSijvFhYsQ_R* zUfYYw;K1AIt0#|@7A6AnT6oyiaW*bIze%_4W6BKup&l!@p^tT4jkf#=FOV4$#a6HB zab&tq*43A=e~wDBc3n*F0gkN;B z4Mut-jm{5;iHt2LX_g-=?HEa|!!c6kq6g1@;}e$?-6K|^iy>8?91Jn^Cc0ckCGt1> z1^JbZrCCd_n~gKoXkIcMLN+NqGHmU(0$H;_$W`d7Ju?vHiPNaq&2#y-QJ##d&@8-1 zQ&AQKXDenEod|o*tODz%|1^%JO*X=Bd*2dp?hlN;kmx5MAD-v+U*L7jC{daW1TB)A zx=1@`UXKEjJB4a*O+JS9`Ux#iG&3s+KRBs@`L@2Adu+mK)zi^y3~ETv#np zUgFtH+NL5sDcme+W~WImcS`cka{(R2pH`zuBwC-O`uspZmnOy*tUbHqdOrt(aE-4p z`-)&`uLJp1#L(sx_>1IM*+ala7&DBTGxU)j!QU(PgyW4$vifZbM+xch2g~jyfM}4C z2mo+|^i=!Ls5k&k_A}N{#oQ+{+4(^V>1bL9hj>ENJC3A5iZ8KmW=#tjS{SOznAJjI z4(73=yWK|qt&mO;W95WzmePGm^Ut#4*v>5@$Z@5W4Bpzzi-c^rL&bz?T{M>aF9Z*_ zb_++SPBQ}(LPS~B7DniqD3?a77DZAbLO6t})N;lgT;b_8O^#{KYI~px6g#IW8*gjC zyQ!pXtJ5LOT7MucGF{5I)xR?VGx5!UR-T=9%+(B|GOE`<68#5oH>ameZlSs-3X{+u zXzLY0s~y{Dwa8ql`?50iY6RdpCgDbTeA)96oh{>cs=bJ!afH0Lx&rz^(@FvLt<<6(eadl5?v7v z%j=gE`G6p6E#=rVsaPz#;JFD)-5_UDS{g#?a}~qW%}My%N!1#>o%>U2wwkW9ETG{) zf$*38aLeCpG(KCRFAA#Bj#9S@&;^3Cm5^_)ZLaP!?~_+G*|yBWRq6NAIi%*&slRNur^E=h^;*m-!De5}eZuhe`mAKj<)82=4i7Oq9Yj~(5f@9L zxEY699`6&(NEv-2$&1^+3<8)umsWp`$7zd&XEyjPAzU*89g{NV18qkFhH(VcjFNm; zPFX6lpWslSMkpf@*Y(90WW1fOk@PnuuQBq<3820L)-X`am*SwN2A;>X&ATD&(Vjru zG8sQa{OF>kl+RG%CV6~_IQUWXP;nEfwBs=W_knXj>Gb5LxY9pKar8h6)r2ev022V9 zcpwG$Ur538`k$ou9|3`TU~c}@*vM$n=tAxKrR||X^?nFB1vzUH1-Y7krfe7jCYa;} zI~=_90>MZ@aF`bYR_}FLpXBu@2Xy$}8$=4Kiv4`;Ab3!jdY{o`UZ7zbIx_&4s|ZAt z2C0{BSkJhN_H`e0Zzki$Y&|)xs)jnWn_P@2?8?a(4cNO&Ky% z*o&6mWX~^IO!g@HG7lPh_VDg(rj&ZzQ#s78@59b@8`_P`-`|~s-Sap(tp}hU7mCU6 z-bHnAc49=gZXIL=e-n3EKdfkZqM$r(L6E>*W-^DmIEh<~ggn2y)|t!|?mR5-J4ZSj z#wrmx7Qxx~Zp+q{bHeJvuV^W_k_;y0Y=$6n% zJX65-DY=*{C_WocCDmoH)1Tr%5hc<$vP*jQj$Gk3SkTnAFsYim0mt;MnxmfBl)rnf zWsQprXWc1!n%J^kULUPu**q~g+kUleLon6SaA)~Q(R}NODImZa!v7Tqfun1987zkeSd+(m5w^*#{K`ot*YxhsNmVpO+2Q-P8nUrbqF+VWz;8riD z1@7a&`0ws`wwDkNg3ub1DzOY?@XlGPFayHj3 zqRx}kecZXWVDX(D!YZ#^+X6kHy@LY%2~QM6o^p62GL}C<@=&L0>yW&}C~CGh6@KuY z3M-umk~u$L@82)?ke5w89%8tN0``=%_C*Di4pa~gh0n;!4}C0+o(|M~jbYCnbgd_E zo5|_9!9}tB^`QyX0`p^KjdVA4fiF>dgtqxEUQ}$oG)2)545h+^`c?#+?oU6U+>cGEZiyYucOjhe#M-3Q(A1@D52U(@W} z*N#hpA-kz8LYc)>v<_>Zrad7^P2tvSb1bPqCPVi;4!^Q|&()QwTjeAD+Ylt^YRK!@ zGwFoON}AFWUmjiTv2A@wm?Ao+)~NsR4j3mygV!E^6*rY!sp@Rj=KVd@O+(QDX9p_E zHLb~d2`XB_D(=_FlscH&QZ3^e{(+_(Ps z4R1OwhjK+OD`{ge$LXvjpc72AAbfJ06*sQr=4ygOOm+_%U=C0<)oPgUhiUt5hvvOU zCutMIWQ))D>FQ5K<<%F>thcjuB*tFBvx~>kUxn9^*Yz;{qRaTUXl9L%;$*?#yX2bO zI$TyzC)SCM_@vNPIM`tWf;ZhvZqMrJ4 zAPR=fYPP8~1H+NmS$)Sy064Ia2^{WTgQ!6V2Y#3hrlV07=6_jL8xs|QOKW1Vsk^`A zwwaiF&?cLJ7)onMyb_VQ^}d_*Sw;nWaGG$=kBbZM2mAU9yo|+uwYLSpjOXJe)E0b^ z5?bsinb_SXt*89OmcBYWoICnUaBUFP;y^FvC>$>c<^C!C(J<{a>Fqu= zqW%?6$4=Fd?Zp3naDay+m_?O%^vBDF_@tiNn5>oFx-8SNgNO9kx{d1B1^U|T? zW^Ca_16csz1qOMPfji{^)%x%yz#sbsAfxRX0mbc1J;!bJ(k4cea$KU5(Uag-5cbd5 zeMU%5x$Gv*#2~^zX{AX1Tf|dj3M@_ml8@)Eh|=VGy5T1DHE_c94=l}OiGxS~?SvB* zn|vK`JYXccIdVQdL5tCT>7wYUjac%cf`2geIrgtZ<(Eh?buY5uMd!#UaMF`sl`irK zvxSYT6c*s{jk8H$+w{hl2$_71taBsBm`IG#*K+Y0h9kB{v6ymXUp=9xzb(opEsb!^ zt+qEn>P}zJPb!JqU$<3p4u4cBm3@NytmZ2^Q3uTU7mgzCmRn3Qv>9zbB?3pTyngdm zXH1(-Ttn>g^k=S1wyRCa#Bzo1Gn!sd;?mi*^RrJ*9E=;D#Mo@P z%o0^}x#4e+gyjtD!jo~~fr{09&uYI`oq)~{8%!gsaeHZuw^qdO zAPS{JLnP<)k#s*IwGseX%juDNQm5<{U`NOj6%hn*q1lvc-Dr3PRQ$gl8ocED5PE+Y(^Ub~&Q12iv5~myf5^C1KHU5(j zw*|9^7;8^bnk7_HHnC(53&UyusoC}#V%ZCOr<#z#oAVCw_C@*8Q^`dJN&{Kzd;bh9 zV#qLe1o0SBO5)bve^)Dyt~xkFhO_9=EeH*bK$4$8l8gXO+aH#gkrR+W#D4+dIATTR zWW_9S4RlQh*6ddBRk+qlJ5I~Cq!e4v99jnP&BYvC?z6_XI5; z9fhudJuE=*5QAM|D9@(NtYd%}*uhuQN7WWh8TPz?Jmw0Ptjl&oUy@+*D|1UAH7Ur_ z$7>?bER(tRWR^{9waK`|C64Q+lO%z@h5lj&5Mm)A*(1IK-^1Uq6Og#qC)22&;t4y6 zQ1M}*CFKYo%uh_X9`%1ZJG+tr zg}SKhn>a*z%oV*br8yK8CT7KY1FKA*6zZ1MkvEVIkSmo z*KNun&1omFc-kWYzo*LJZJ!z}cJcPszShOhi0UC%b!Vn&A*=6y2`%<51}tm2p?i2ciqq+~OEr7tpq=ygvA0s>9 z@5}$Yr}Wn{|4u>vqX+fEal)#T39smHKKTJX( L03c2D_ubzB!4Qw5 literal 0 HcmV?d00001 diff --git a/tests/TEST_INPUTS/results/SynthSAContextSensitivity/BBBaseO-CR-SynthSAContextSensitivity-ContextSense-06eac0edb6/b24ee2c1-fc85-47ba-abbd-90c98e88a37c_config-0_success.zip b/tests/TEST_INPUTS/results/SynthSAContextSensitivity/BBBaseO-CR-SynthSAContextSensitivity-ContextSense-06eac0edb6/b24ee2c1-fc85-47ba-abbd-90c98e88a37c_config-0_success.zip new file mode 100644 index 0000000000000000000000000000000000000000..02e15588784a5e567cd9ddedaddada2d65970d3c GIT binary patch literal 5145 zcmbW*Ra6w-z6bC@7`jV31_9|t>F#c6K}O123t12M{S05D96KK{}*k=C5=k zLtM_e>z;ES&RyrOyZ6Jcwf1kneb#py>R{tg0{{Sg09n|jdAqvs;2$6Wz<>t;NZs#x zIrzPB0Xx}?O4$2=oqfQ*zVH~&=J^XwHgI&F{ zXM)73t(fo*n{=HRmA_^1$ESmXbBZVC8Gcrvh*i#^84zkQ5mhI9AO37kTvre#rxUQ%x{ZRqiyM;ZudkrmImU0=_-fPb0U*Hko)!!a(>GeRfNXe-iFzQ3cE zO5(9#!VPhf=R5DQkJF1GIvP)K#CaDyrtNZRqzR4PKklWkCbX-<%XWQ2Le3Q(y!1|L zj{_zZdpRFX-SmO8TYc{WDJ(~@x>2@rn+)l)bCTDF+<=pOI^OPp8Ru=B^=lZ%9n%RT zUxdO&jQ|J-W@x-CjluUPATK#qt|ad?1NPWo+mhV*roAL)YNV{?nbXU(J_R()Z70L; zV-D=jBfqLINfk*ojuT9%xSF{#gIng22GC$_x83&8eql6ZrJ+R@hQf^s7CZ;~IleP< zgkXD>7_7Cl5Ue+*{>+et%!FKjA*7PNu=%y`;u+qY&=%)4Qmm6ItvVlwO>4mrRqw94 zIQX;w59@PwJ$FW)rPo?SL2Rwg*4*LdJA!I*s|*v58~Uh9FU;lhF>Rutfznl4uWEs` zusb($-aciC4!@49Y2U6mE*8yrTg@a{#*jjl7P#qvijxwNtsc`8s`Q~4r^jH7Z390^ zTYKy(Q$+~y6VfG~u*ZVv<;KUnbOj0kJw7R+TY$tq3zid}(kdCwp|ZJYtJb&m^#*c= zr?o!}b6&}Rpe~bVKEtUsx3;&SLBMA*g{2*rZp6VT^C2vaX#Qpk_M^-8l#4YG?gJqk z;_n>FnhB8r7cfuYn^^jo8ewuk0C|=o#j<+})zC`Zq)@b$e-!tb+1#QMF;0~*Gr{T$ zZ|dt=|I^d<(5(QRuj&o^2^)MK?-}(~^-9IW7rJmDd@Z9ddBknEEG5`cCM>?Us-=U9 zJ#gG7{Ub8EYTS`aw1jJp+w0@If?nmNsj9E#UF^{l0TYRxXK77*1zV?BlxjjqWTwHz zYcA`Z9M#A#E+0B^v3m`bh?;26gdTLW;`hixcvs2$#mGF*)TzTD17G-;51+swL*L&+DO~#m;#@>Edd#>h_Kc7(P+Ue%?0CMb{%B! zVv{`qiwIH*WPQJJoLOPl4Tw%2!o+yI1MXJmd#9SBDf+SHcJSmBa!YmI@0Djo8yA^! zBoqrC*w;tmtIMnHc*ViSbeV@1YTK#Zqyv3EczDV0pLGuXl_t(<_;q?1CF3%xRr$n= z`-09eQ`zZB&GtYik7`N3<&tFgtwW(FRGx3)C7*W*sI)4h>=?*oA?($Ze^*ApD&q1U?`(03hgfq}=iBvJT2aeK z&HUgzSJwzk68(@Yl^*Pom{Qfy0eetKgH#;S&Y{6~%r7$lls!`?u$!%_mcKA0CRyS8 zu`lCzOGC=6Auq)n<55=KqMs-qY`oV8VdbawU)&e69^qa$WzcueV?g}OUFet~t4Tj$ zpD0q@4kb=fTXRccLP_T^zH(G5v*&ZhGo$>$t^C}5DVI1w4yRRt{&XHI!(JJEpc3U& zaV##1e2zh1$;G0Y=d`$cxXg2We4cO4;_%`nlR#yj8WqfP50w-y6&Do8(0o%So?cw; zv+fqN0s_OlC@ffq>ztKAd3nHRw;?RM^`GBH*y_ivQr(#fc2h=v|6EsaKUY1o*sz;j z?o|Utgmm3Jci06Fj-1?_lNtWQlSA00`OhLDJw{vrfbQNC;eYW&MDib={0Bxto>B$V z;TsXmulopH*n<}Q}`k8eCe7nJhzIM9ZsJ8GLcnu;mvRuS~o`> zTgTd@-)O8-mbg{8z_@GIN(4~0oc*YX=zGX%BUjTb3&LRfw%mO&jZ^Zs^tOG5G=JEp zSGT#`3Bc~f#LPr>pJA(k5L{2UaZ1hlVUyzCO|Nb0vCsU}Vdyi;bUN80!De@gj^D*4 zON)5vuL7{AI7hPLtLZXkZcUJ1>L51xoUFVfu*@K}8$-ZtF*bamup|xJrmaumulXxS zvx@EGAF!ybsmB*L5aVwk>aOy{{k(?CUc(Vpmm zN4Y}XiDMGRq!4r&T4tv|)LVbxS$^F7ubdI!>9d!3j-9x#aO~=Mbj{w(uKn)c(A=-Y zsR6p?P5LG|hH%g(Q*IV)Bt|^6Nc{S(Feqp5D3?TpBPc@L1c@26_ft^ni zUT7=e=gOA%7oPZUZz+VkIThpHnEz;oVKJ=YoQBc#Afn60>bIEo=l1&-m?O8C7xhNS zL`tmFE8j?Z*abpEA(f+S>Q=H&BX_p6n58ka9x+OXx?ah2xzdJx{&x6rRiV*z3aCe? zQ(-a#h)@F4OR{usB^;Tg5BVYd|FTd2Isz$ob_aQom&_U*N?0vwh9)$G`vklJ&ZL@$ zS}C_tRx$5=bS-*6XR*j0Y{Z)Jx~N29u$8T27B@sZHE$~Ed#{nr*(fL3UsNWYy1`Hj zJ;t&tV-X2zenATK3{%^POj3fv=2^5QM~wZI*H3wLvt(Ne0c=A4lW667y%~)l3`!!L zN#!`S8`d6KPESz^g9|gafe-5|PSu0$=s2b{b28Tyl;0*!eprGL;UVk8y+#&4BT$}3 zLurIX5*`=#AuYzWoYPX3~W0-vuv7Hx`9mD-awpC-M|x>}3CK!KgFXX6fxm$;O`6HT7->?1j^16d$)h~7i+LOj!Yn&YzXR^*Xe|4|ZAt6L} zgs!k^T{gPckYHn9VO$?4IyngijWe-STh0_1zf|MgGLwJ*TD&j`!jI`eBSR3mhEEOA7t&2T{sk;VN-YGlQU%dqT?r%!}tW(j;RrLc-u=)fr6 z+nvC2eUA^T@u1%>TT0Su9#Jalr&Y-0L7OH$E!CZ)Zvo<)QK@p6kmvtpV{q;`z9m2; z@rL;mTkBKS{e5|f^|zT1$~EoZ8`Hj9k@#>cvRa%*WT0Qs&%xE^MVi^e8p)^gYddup zJT}e^qi-D9jYxKnrxSQL^Asd&DbK`2r{Qmq5@Gd>3rGiyEbflSs*KzZfuOgUtyu+u@GOqN z*4jq(JX~cRJ6+g;pJKijkF~&sP2d^x?M}P{*#oqx?}GayQXYk7gW+osK6ng!3GeCv za=1s`5BS*57^Zu@;PZ^KFQjH@`c+r27H#OdxC!oL>V#r?lAOgfE- zk}&KHL(<#}DL8P;hoy6LY%pCU$yzxSnT5@T^%Uuq^Aql zKlN=?N2bu0fxYZ{Wyiz!*OxHMWfJN|qBc#rRJdMyMeagAfvG}e1xZRwmYwxzyu-Nq zs*XsLa|&m76X~SISS-7sveXMJZVyvT_4?vYRivp3Rpdo4)PQyOJYpuQa#5#gqRTmI z?;J`vjXNqG=z;K6Q_RghbH#o7lR*em<^6?8K#o+_mc91WbMlm5EZKC%Y`NAJT9~)j z*DK^T^mgP$Nf0p$YU41=Tp-EPo>zP{NqW>s2m9iKfCrM-ipOGwI{7D7=jdtd#FhPS z5X!}&H)})mCYj^i`0E zmy0XZv132EQ0h^mS2;&%NvdM&1!}z-rvKL08s`lr6m#-_;?)=0r?%mk{d#~us=R!$ zNTr)+0$#1~R;p%_j7KOCrtCeSpt|1{;~hnArG(1K2n!WdqHw&E0olpNdI(b?*p0dw zhhTaTuLri&S-R6Q(!@+Ht2H3?Z6QL~R@o(cn8Zk36nVpJm-w1(-BJ9UfS-++{toiX zp8hjy(T{^iNX2c1AmZED2m9&S&)in zj_Om>W0NVv=ifh#Dmxmu4iih81(M1a1{88b9|)ko{*_-IUo6U8Ua5|KrPL?4yu6nh ztA~==S~2xait^X|L*R3g;J5d)3TI1>KQCLD_Si;YLR?^)9;q|;PFVPk+gJjb?#xF< z4Pch(8YaW>G^Zy<3olhJoaF*H)cQ3UjD$~nte#7}tbQP*T*~?EYmz9wcJ`E3(Jx(a zm2~XNxNCL$$$`Ai7%f|P1Wozgw!IP9V)4tFcy)h_=^h5!MHL8BWV4UaI~rLhczhkL zXsIK6_vFDgmWLuay)6zPTJw^0 zeItK}gG7<;d6Rg1cY*_Rr26lScEgyr-}_JESa6aDm&5lqOvw4-4uz z_omb}_ycB-rGxC)I}+sl-*^^VD(P1FyRhQ_*qBEM-hvc4r!yn?``{$_FXHbA6hmnv zcpN%E*lt*_^fx-h#!spyqp>plKCKw17C#x%a(iWdDlNrMd?h2d2Vef=^eYQG(~e!!{A@ zO@gYsRiM3r5&l|Q;rdX22}#aDyy9SAay~0nIO;THg96ouh+)LGl)rl}-DZs^gvtGV zF9+5GYV800)9ap(zqg9WzYYKIFR*_t^Y60Ie-{A&Dk2T<6Vm^b`M)Nkf4%+R0r=0` c+ll`JhlV=9`v(93*!P_r3jjEw{M+ik0I$%29smFU literal 0 HcmV?d00001 diff --git a/tests/data/test_report.py b/tests/data/test_report.py index fdb8d1196..f9e5cf9ab 100644 --- a/tests/data/test_report.py +++ b/tests/data/test_report.py @@ -156,6 +156,14 @@ def test_get_uuid(self): self.assertEqual(self.report_filename.uuid, self.correct_UUID) self.assertRaises(ValueError, lambda: self.broken_report_filename.uuid) + def test_experiment_shorthand_parsing_with_path_in_name(self) -> None: + """Checks that we correctly parse the experiment shorthand also in cases + where we have a path as part of the filename.""" + prefixed = ReportFilename( + "/tmp/foobar/" + self.report_filename.filename + ) + self.assertEqual(prefixed.experiment_shorthand, "CRE") + class TestConfigReportFilename(unittest.TestCase): """Test configuration specific ReportFilename functionality.""" diff --git a/tests/paper_mgmt/test_case_study.py b/tests/paper_mgmt/test_case_study.py index 01e22ab56..7a70b3325 100644 --- a/tests/paper_mgmt/test_case_study.py +++ b/tests/paper_mgmt/test_case_study.py @@ -239,7 +239,8 @@ def test_get_newest_result_files_for_case_study_with_empty_res_dir( UnitTestFixtures.PAPER_CONFIGS, UnitTestFixtures.RESULT_FILES ) def test_get_newest_result_files_for_case_study_with_config(self) -> None: - """Check that when we have two files, the newes one get's selected.""" + """Check that when we have two files that differ in their config id, + both get selected.""" vara_cfg()['paper_config']['current_config'] = "test_config_ids" load_paper_config() @@ -273,7 +274,56 @@ def test_get_newest_result_files_for_case_study_with_config(self) -> None: self.assertEqual(newest_res_filenames[0].config_id, 0) self.assertEqual(newest_res_filenames[1].config_id, 1) - self.assertEqual(len(newest_res_filenames), 2) + self.assertEqual(newest_res_filenames[2].config_id, 0) + self.assertEqual(newest_res_filenames[3].config_id, 1) + self.assertEqual(len(newest_res_filenames), 4) + + @run_in_test_environment( + UnitTestFixtures.PAPER_CONFIGS, UnitTestFixtures.RESULT_FILES + ) + def test_get_newest_result_files_for_case_study_with_diff_exp(self) -> None: + """Check that when we have two files that differ in their experiment + shorthand, both get selected.""" + vara_cfg()['paper_config']['current_config'] = "test_config_ids" + load_paper_config() + + config_0_file = ReportFilename( + "BBBase-CR-SynthSAContextSensitivity-ContextSense-06eac0edb6/" + "b24ee2c1-fc85-47ba-abbd-90c98e88a37c_config-0_success.zip" + ) + config_1_file = ReportFilename( + "BBBaseO-CR-SynthSAContextSensitivity-ContextSense-06eac0edb6/" + "b24ee2c1-fc85-47ba-abbd-90c98e88a37c_config-0_success.zip" + ) + + now = datetime.now().timestamp() + file_path_0 = Path( + str(vara_cfg()['result_dir']) + ) / 'SynthSAContextSensitivity' / config_0_file.filename + os.utime(file_path_0, (now, now)) + + file_path_1 = Path( + str(vara_cfg()['result_dir']) + ) / 'SynthSAContextSensitivity' / config_1_file.filename + os.utime(file_path_1, (now, now)) + + newest_res_files = MCS.get_newest_result_files_for_case_study( + get_paper_config().get_case_studies('SynthSAContextSensitivity')[0], + Path(vara_cfg()['result_dir'].value), CR + ) + + newest_res_files.sort(reverse=True) + newest_res_filenames = [ReportFilename(x) for x in newest_res_files] + + self.assertEqual( + newest_res_filenames[0].experiment_shorthand, "BBBaseO" + ) + self.assertEqual( + newest_res_filenames[1].experiment_shorthand, "BBBaseO" + ) + self.assertEqual(newest_res_filenames[2].experiment_shorthand, "BBBase") + self.assertEqual(newest_res_filenames[3].experiment_shorthand, "BBBase") + self.assertEqual(len(newest_res_filenames), 4) def test_get_case_study_file_name_filter_empty(self) -> None: """Check that we correctly handle case study filter generation even if diff --git a/varats-core/varats/report/report.py b/varats-core/varats/report/report.py index 0a649a7dc..ccbffcdbc 100644 --- a/varats-core/varats/report/report.py +++ b/varats-core/varats/report/report.py @@ -311,7 +311,7 @@ def experiment_shorthand(self) -> str: the experiment shorthand from a result file """ if (match := ReportFilename.__RESULT_FILE_REGEX.search(self.filename)): - return match.group("experiment_shorthand") + return match.group("experiment_shorthand").split('/')[-1] raise ValueError(f'File {self.filename} name was wrongly formatted.') diff --git a/varats/varats/paper_mgmt/case_study.py b/varats/varats/paper_mgmt/case_study.py index 823c7f154..556441cde 100644 --- a/varats/varats/paper_mgmt/case_study.py +++ b/varats/varats/paper_mgmt/case_study.py @@ -301,7 +301,7 @@ def get_newest_result_files_for_case_study( Returns: list of result file paths """ - files_to_store: tp.Dict[tp.Tuple[ShortCommitHash, tp.Optional[int]], + files_to_store: tp.Dict[tp.Tuple[ShortCommitHash, str, tp.Optional[int]], Path] = {} result_dir /= case_study.project_name @@ -319,16 +319,23 @@ def get_newest_result_files_for_case_study( ) if case_study.has_revision(commit_hash) and config_id_matches: - current_file = files_to_store.get((commit_hash, config_id), - None) + current_file = files_to_store.get( + (commit_hash, report_file.experiment_shorthand, config_id), + None + ) if current_file is None: - files_to_store[(commit_hash, config_id)] = opt_res_file + files_to_store[( + commit_hash, report_file.experiment_shorthand, config_id + )] = opt_res_file else: if ( current_file.stat().st_mtime < opt_res_file.stat().st_mtime ): - files_to_store[(commit_hash, config_id)] = opt_res_file + files_to_store[( + commit_hash, report_file.experiment_shorthand, + config_id + )] = opt_res_file return list(files_to_store.values()) From abf1d9200d68d39e0260c3c7465a09a07641593c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20B=C3=B6hm?= Date: Wed, 27 Sep 2023 11:36:56 +0200 Subject: [PATCH 7/8] Adds functionality for patch-based configuration (#839) Compile-time configurable projects can now be configured using patches. Therefore, patches can specify a list of feature_tags. Case studies can contain a configuration map with PatchConfigurations that contains a mapping from config id to a list of feature tags associated with this configuration. Configuration patches are selected by matching their feature tags to the feature tags of the given configuration. Since there are multiple types of configuration maps now, their type must be specified using the key config_type in case study files. --- .../configuration_specific_experiments.rst | 1 + .../SynthSAContextSensitivity_0.case_study | 1 + .../test_config_ids/xz_0.case_study | 1 + tests/paper/test_case_study.py | 1 + tests/provider/test_patch_provider.py | 125 +++++++++++++++--- varats-core/varats/base/configuration.py | 46 +++++++ .../varats/experiment/experiment_util.py | 89 ++++++++++--- .../varats/experiment/workload_util.py | 39 ++---- .../varats/mapping/configuration_map.py | 1 + varats-core/varats/paper/case_study.py | 18 ++- varats-core/varats/project/project_util.py | 33 ----- varats-core/varats/project/varats_command.py | 92 +++++++++++++ .../varats/provider/patch/patch_provider.py | 67 +++++++--- varats-core/varats/utils/filesystem_util.py | 3 +- varats/varats/projects/c_projects/xz.py | 4 +- 15 files changed, 404 insertions(+), 117 deletions(-) create mode 100644 varats-core/varats/project/varats_command.py diff --git a/docs/source/tutorials/configuration_specific_experiments.rst b/docs/source/tutorials/configuration_specific_experiments.rst index 16479538c..77f804b6a 100644 --- a/docs/source/tutorials/configuration_specific_experiments.rst +++ b/docs/source/tutorials/configuration_specific_experiments.rst @@ -21,6 +21,7 @@ One just needs to extend the case-study file of a project with a yaml document t .. code-block:: yaml --- + config_type: PlainCommandlineConfiguration 0: '["--foo", "--bar"]' 1: '["--foo"]' ... diff --git a/tests/TEST_INPUTS/paper_configs/test_config_ids/SynthSAContextSensitivity_0.case_study b/tests/TEST_INPUTS/paper_configs/test_config_ids/SynthSAContextSensitivity_0.case_study index 2a872480b..dd8b44c42 100644 --- a/tests/TEST_INPUTS/paper_configs/test_config_ids/SynthSAContextSensitivity_0.case_study +++ b/tests/TEST_INPUTS/paper_configs/test_config_ids/SynthSAContextSensitivity_0.case_study @@ -14,6 +14,7 @@ stages: version: 0 ... --- +config_type: PlainCommandlineConfiguration 0: '["--compress", "--mem", "10", "8"]' 1: '["--compress", "--mem", "300", "8"]' ... diff --git a/tests/TEST_INPUTS/paper_configs/test_config_ids/xz_0.case_study b/tests/TEST_INPUTS/paper_configs/test_config_ids/xz_0.case_study index e1101d9be..a544fbfbc 100644 --- a/tests/TEST_INPUTS/paper_configs/test_config_ids/xz_0.case_study +++ b/tests/TEST_INPUTS/paper_configs/test_config_ids/xz_0.case_study @@ -10,5 +10,6 @@ stages: config_ids: [1] version: 0 --- +config_type: PlainCommandlineConfiguration 0: '["--foo", "--bar"]' 1: '["--foo"]' \ No newline at end of file diff --git a/tests/paper/test_case_study.py b/tests/paper/test_case_study.py index 6d8412ee9..6a18fbbf3 100644 --- a/tests/paper/test_case_study.py +++ b/tests/paper/test_case_study.py @@ -48,6 +48,7 @@ commit_id: 494 ... --- +config_type: ConfigurationImpl 0: '{"foo": true, "bar": false, "bazz": "bazz-value", "buzz": "None"}' 1: '{}' 2: '{}' diff --git a/tests/provider/test_patch_provider.py b/tests/provider/test_patch_provider.py index 9acebb32e..9b55604e7 100644 --- a/tests/provider/test_patch_provider.py +++ b/tests/provider/test_patch_provider.py @@ -184,57 +184,120 @@ def setUpClass(cls) -> None: "Test-ABCD", "", path=Path("test.patch"), - tags={"A", "B", "C", "D"} + tags={"A", "B", "C", "D"}, + feature_tags={"F_A", "F_B", "F_C", "F_D"} ), - Patch("TEST", "Test-A", "", path=Path("test.patch"), tags={"A"}), - Patch("TEST", "Test-B", "", path=Path("test.patch"), tags={"B"}), - Patch("TEST", "Test-C", "", path=Path("test.patch"), tags={"C"}), - Patch("TEST", "Test-D", "", path=Path("test.patch"), tags={"D"}), Patch( - "TEST", "Test-AB", "", path=Path("test.patch"), tags={"A", "B"} + "TEST", + "Test-A", + "", + path=Path("test.patch"), + tags={"A"}, + feature_tags={"F_A"} + ), + Patch( + "TEST", + "Test-B", + "", + path=Path("test.patch"), + tags={"B"}, + feature_tags={"F_B"} + ), + Patch( + "TEST", + "Test-C", + "", + path=Path("test.patch"), + tags={"C"}, + feature_tags={"F_C"} ), Patch( - "TEST", "Test-AC", "", path=Path("test.patch"), tags={"A", "C"} + "TEST", + "Test-D", + "", + path=Path("test.patch"), + tags={"D"}, + feature_tags={"F_D"} + ), + Patch( + "TEST", + "Test-AB", + "", + path=Path("test.patch"), + tags={"A", "B"}, + feature_tags={"F_A", "F_B"} + ), + Patch( + "TEST", + "Test-AC", + "", + path=Path("test.patch"), + tags={"A", "C"}, + feature_tags={"F_A", "F_C"} ), Patch( - "TEST", "Test-AD", "", path=Path("test.patch"), tags={"A", "D"} + "TEST", + "Test-AD", + "", + path=Path("test.patch"), + tags={"A", "D"}, + feature_tags={"F_A", "F_D"} ), Patch( - "TEST", "Test-BC", "", path=Path("test.patch"), tags={"B", "C"} + "TEST", + "Test-BC", + "", + path=Path("test.patch"), + tags={"B", "C"}, + feature_tags={"F_B", "F_C"} ), Patch( - "TEST", "Test-BD", "", path=Path("test.patch"), tags={"B", "D"} + "TEST", + "Test-BD", + "", + path=Path("test.patch"), + tags={"B", "D"}, + feature_tags={"F_B", "F_D"} ), Patch( - "TEST", "Test-CD", "", path=Path("test.patch"), tags={"C", "D"} + "TEST", + "Test-CD", + "", + path=Path("test.patch"), + tags={"C", "D"}, + feature_tags={"F_C", "F_D"} ), Patch( "TEST", "Test-ABC", "", path=Path("test.patch"), - tags={"A", "B", "C"} + tags={"A", "B", "C"}, + feature_tags={"F_A", "F_B", "F_C"} ), Patch( "TEST", "Test-ABD", "", path=Path("test.patch"), - tags={"A", "B", "D"} + tags={"A", "B", "D"}, + feature_tags={"F_A", "F_B", "F_D"} ), Patch( "TEST", "Test-ACD", "", path=Path("test.patch"), - tags={"A", "C", "D"} + tags={"A", "C", "D"}, + feature_tags={"F_A", "F_C", "F_D"} ), Patch( "TEST", "Test-BCD", "", path=Path("test.patch"), - tags={"B", "C", "D"} + tags={"B", "C", "D"}, + feature_tags={"F_B", "F_C", "F_D"} ), } @@ -311,6 +374,38 @@ def test_any_of_multiple_tags(self): for patch in patches: any([tag in patch.tags for tag in tags]) + def test_all_of_single_feature_tag(self): + for tag in {"F_A", "F_B", "F_C", "F_D"}: + patches = self.patchSet.all_of_features([tag]) + self.assertEqual(8, len(patches)) + + def test_all_of_multiple_feature_tags(self): + tags_count = {("F_A", "F_B"): 4, + ("F_C", "F_B"): 4, + ("F_D", "F_B"): 4, + ("F_A", "F_B", "F_C"): 2, + ("F_A", "F_B", "F_C", "F_D"): 1} + + for tags in tags_count: + patches = self.patchSet.all_of_features(tags) + self.assertEqual(tags_count[tags], len(patches)) + + def test_any_of_single_feature_tag(self): + for tag in {"F_A", "F_B", "F_C", "F_D"}: + patches = self.patchSet.any_of_features([tag]) + self.assertEqual(8, len(patches)) + + def test_any_of_multiple_feature_tags(self): + tags_count = {("F_A", "F_B"): 12, + ("F_C", "F_B"): 12, + ("F_D", "F_B"): 12, + ("F_A", "F_B", "F_C"): 14, + ("F_A", "F_B", "F_C", "F_D"): 15} + + for tags in tags_count: + patches = self.patchSet.any_of_features(tags) + self.assertEqual(tags_count[tags], len(patches)) + def test_patchset_intersection(self): patches = self.patchSet["A"] & self.patchSet["B"] diff --git a/varats-core/varats/base/configuration.py b/varats-core/varats/base/configuration.py index 0c667dd2e..cdf6cb8e5 100644 --- a/varats-core/varats/base/configuration.py +++ b/varats-core/varats/base/configuration.py @@ -414,3 +414,49 @@ def get_config_value(self, option_name: str) -> tp.Optional[tp.Any]: def unfreeze(self) -> Configuration: return self + + +class PatchConfiguration(Configuration): + """Configuration class for projects where configuring is done by applying a + patch.""" + + def __init__(self, patch_feature_tags: tp.Set[str]): + self.__patch_feature_tags: tp.Set[ConfigurationOption] = { + ConfigurationOptionImpl(tag, tag) for tag in patch_feature_tags + } + + @staticmethod + def create_configuration_from_str(config_str: str) -> Configuration: + patch_feature_tags = json.loads(config_str) + return PatchConfiguration(patch_feature_tags) + + def add_config_option(self, option: ConfigurationOption) -> None: + self.__patch_feature_tags.add(option) + + def set_config_option(self, option_name: str, value: tp.Any) -> None: + self.__patch_feature_tags = { + option for option in self.__patch_feature_tags + if option.name != option_name + } + self.add_config_option(ConfigurationOptionImpl(option_name, value)) + + def get_config_value(self, option_name: str) -> tp.Optional[tp.Any]: + filtered_options = filter( + lambda option: (option.name == option_name), + self.__patch_feature_tags + ) + return any(filtered_options) + + def options(self) -> tp.List[ConfigurationOption]: + return list(self.__patch_feature_tags) + + def dump_to_string(self) -> str: + return ", ".join( + map(lambda option: str(option.value), self.__patch_feature_tags) + ) + + def freeze(self) -> FrozenConfiguration: + return FrozenConfiguration(deepcopy(self)) + + def unfreeze(self) -> Configuration: + return self diff --git a/varats-core/varats/experiment/experiment_util.py b/varats-core/varats/experiment/experiment_util.py index 110782502..bad60ba6f 100644 --- a/varats-core/varats/experiment/experiment_util.py +++ b/varats-core/varats/experiment/experiment_util.py @@ -10,7 +10,6 @@ from collections import defaultdict from pathlib import Path from types import TracebackType -from typing import Protocol, runtime_checkable from benchbuild import source from benchbuild.experiment import Experiment @@ -23,11 +22,17 @@ from plumbum.commands.base import BoundCommand import varats.revision.revisions as revs -from varats.base.configuration import PlainCommandlineConfiguration +from varats.base.configuration import ( + PlainCommandlineConfiguration, + PatchConfiguration, + Configuration, +) +from varats.experiment.steps.patch import ApplyPatch from varats.paper.paper_config import get_paper_config from varats.project.project_util import ProjectBinaryWrapper from varats.project.sources import FeatureSource from varats.project.varats_project import VProject +from varats.provider.patch.patch_provider import PatchSet, PatchProvider from varats.report.report import ( BaseReport, FileStatusExtension, @@ -696,20 +701,12 @@ def get_current_config_id(project: VProject) -> tp.Optional[int]: return None -def get_extra_config_options(project: VProject) -> tp.List[str]: - """ - Get extra program options that were specified in the particular - configuration of \a Project. - - Args: - project: to get the extra options for - - Returns: - list of command line options as string - """ +def get_config( + project: VProject, config_type: tp.Type[Configuration] +) -> tp.Optional[Configuration]: config_id = get_current_config_id(project) if config_id is None: - return [] + return None paper_config = get_paper_config() case_studies = paper_config.get_case_studies(cs_name=project.name) @@ -722,14 +719,68 @@ def get_extra_config_options(project: VProject) -> tp.List[str]: case_study = case_studies[0] config_map = load_configuration_map_for_case_study( - paper_config, case_study, PlainCommandlineConfiguration + paper_config, case_study, config_type ) config = config_map.get_configuration(config_id) - if config is None: - raise AssertionError( - "Requested config id was not in the map, but should be" - ) + return config + + +def get_extra_config_options(project: VProject) -> tp.List[str]: + """ + Get extra program options that were specified in the particular + configuration of \a Project. + + Args: + project: to get the extra options for + Returns: + list of command line options as string + """ + config = get_config(project, PlainCommandlineConfiguration) + if not config: + return [] return list(map(lambda option: option.value, config.options())) + + +def get_config_patches(project: VProject) -> PatchSet: + """ + Get required patches for the particular configuration of \a Project. + + Args: + project: to get the patches for + + Returns: + list of patches + """ + config = get_config(project, PatchConfiguration) + if not config: + return PatchSet(set()) + + patch_provider = PatchProvider.create_provider_for_project(project) + revision = ShortCommitHash(project.revision.primary.version) + feature_tags = {opt.value for opt in config.options()} + patches = patch_provider.get_patches_for_revision(revision).all_of_features( + feature_tags + ) + + return patches + + +def get_config_patch_steps(project: VProject) -> tp.MutableSequence[Step]: + """ + Get a list of actions that apply all configuration patches to the project. + + Args: + project: the project to be configured + + Returns: + the actions that configure the project + """ + return list( + map( + lambda patch: ApplyPatch(project, patch), + get_config_patches(project) + ) + ) diff --git a/varats-core/varats/experiment/workload_util.py b/varats-core/varats/experiment/workload_util.py index 4566e2ee7..38b82720f 100644 --- a/varats-core/varats/experiment/workload_util.py +++ b/varats-core/varats/experiment/workload_util.py @@ -19,8 +19,12 @@ Command, ) -from varats.experiment.experiment_util import get_extra_config_options +from varats.experiment.experiment_util import ( + get_extra_config_options, + get_config_patches, +) from varats.project.project_util import ProjectBinaryWrapper +from varats.project.varats_command import VCommand from varats.project.varats_project import VProject from varats.report.report import KeyedReportAggregate, ReportTy from varats.utils.exceptions import auto_unwrap @@ -93,34 +97,19 @@ def workload_commands( ) ] - # Filter commands that have required args set. + # Filter commands that have required args and patches set. extra_options = set(get_extra_config_options(project)) + patches = get_config_patches(project) - def requires_any_filter(prj_cmd: ProjectCommand) -> bool: - if hasattr( - prj_cmd.command, "requires_any" - ) and prj_cmd.command.requires_any: - args = set(prj_cmd.command._args).union(extra_options) - return bool(args.intersection(prj_cmd.command.requires_any)) - return True - - def requires_all_filter(prj_cmd: ProjectCommand) -> bool: - if hasattr( - prj_cmd.command, "requires_all" - ) and prj_cmd.command.requires_all: - args = set(prj_cmd.command._args).union(extra_options) - return bool(prj_cmd.command.requires_all.issubset(args)) + def filter_by_config(prj_cmd: ProjectCommand) -> bool: + if isinstance(prj_cmd.command, VCommand): + return prj_cmd.command.can_be_executed_by(extra_options, patches) return True - available_cmds = filter( - requires_all_filter, filter(requires_any_filter, project_cmds) - ) - - return list( - filter( - lambda prj_cmd: prj_cmd.path.name == binary.name, available_cmds - ) - ) + return [ + cmd for cmd in project_cmds + if cmd.path.name == binary.name and filter_by_config(cmd) + ] def create_workload_specific_filename( diff --git a/varats-core/varats/mapping/configuration_map.py b/varats-core/varats/mapping/configuration_map.py index f472c7d00..71a71122e 100644 --- a/varats-core/varats/mapping/configuration_map.py +++ b/varats-core/varats/mapping/configuration_map.py @@ -141,6 +141,7 @@ def create_configuration_map_from_yaml_doc( """ new_config_map = ConfigurationMap() + yaml_doc.pop("config_type", None) for config_id in sorted(yaml_doc): parsed_config = concrete_config_type.create_configuration_from_str( diff --git a/varats-core/varats/paper/case_study.py b/varats-core/varats/paper/case_study.py index 627e96d01..3fb087596 100644 --- a/varats-core/varats/paper/case_study.py +++ b/varats-core/varats/paper/case_study.py @@ -169,7 +169,7 @@ def get_config_ids_for_revision(self, revision: CommitHash) -> tp.List[int]: Returns a list of all configuration IDs specified for this revision. Args: - revision: i.e., a commit hash registed in this ``CSStage`` + revision: i.e., a commit hash registered in this ``CSStage`` Returns: list of config IDs """ @@ -580,11 +580,19 @@ def load_configuration_map_from_case_study_file( version_header.raise_if_not_type("CaseStudy") version_header.raise_if_version_is_less_than(1) - next(documents) # Skip case study yaml-doc + next(documents) # skip case study document + try: + while True: + document = next(documents) - return create_configuration_map_from_yaml_doc( - next(documents), concrete_config_type - ) + if document["config_type"] == concrete_config_type.__name__: + break + + return create_configuration_map_from_yaml_doc( + document, concrete_config_type + ) + except StopIteration: + return ConfigurationMap() def store_case_study(case_study: CaseStudy, case_study_location: Path) -> None: diff --git a/varats-core/varats/project/project_util.py b/varats-core/varats/project/project_util.py index 3028c438b..a4c27d74d 100644 --- a/varats-core/varats/project/project_util.py +++ b/varats-core/varats/project/project_util.py @@ -7,7 +7,6 @@ import benchbuild as bb import pygit2 -from benchbuild.command import Command from benchbuild.source import Git from benchbuild.utils.cmd import git from plumbum import local @@ -383,35 +382,3 @@ def copy_renamed_git_to_dest(src_dir: Path, dest_dir: Path) -> None: for name in dirs: if name == ".gitted": os.rename(os.path.join(root, name), os.path.join(root, ".git")) - - -class VCommand(Command): # type: ignore [misc] - """ - Wrapper around benchbuild's Command class. - - Attributes: - requires_any: sufficient args that must be available for successful execution. - requires_all: all args that must be available for successful execution. - """ - - _requires: tp.Set[str] - - def __init__( - self, - *args: tp.Any, - requires_any: tp.Optional[tp.Set[str]] = None, - requires_all: tp.Optional[tp.Set[str]] = None, - **kwargs: tp.Union[str, tp.List[str]], - ) -> None: - - super().__init__(*args, **kwargs) - self._requires_any = requires_any if requires_any else set() - self._requires_all = requires_all if requires_all else set() - - @property - def requires_any(self) -> tp.Set[str]: - return self._requires_any - - @property - def requires_all(self) -> tp.Set[str]: - return self._requires_all diff --git a/varats-core/varats/project/varats_command.py b/varats-core/varats/project/varats_command.py new file mode 100644 index 000000000..314a1ee55 --- /dev/null +++ b/varats-core/varats/project/varats_command.py @@ -0,0 +1,92 @@ +"""Custom version of benchbuild's Command for use with the VaRA-Tool-Suite.""" +import typing as tp + +from benchbuild.command import Command + +if tp.TYPE_CHECKING: + import varats.provider.patch.patch_provider as patch_provider + + +class VCommand(Command): # type: ignore [misc] + """ + Wrapper around benchbuild's Command class. + + Attributes: + requires_any_args: any of these command line args must be available for + successful execution. + requires_all_args: all of these command line args must be available for + successful execution. + requires_any_patch: any of these patch feature-tags must be available for + successful execution. + requires_all_patch: all of these patch feature-tags must be available for + successful execution. + """ + + _requires: tp.Set[str] + + def __init__( + self, + *args: tp.Any, + requires_any_args: tp.Optional[tp.Set[str]] = None, + requires_all_args: tp.Optional[tp.Set[str]] = None, + requires_any_patch: tp.Optional[tp.Set[str]] = None, + requires_all_patch: tp.Optional[tp.Set[str]] = None, + **kwargs: tp.Union[str, tp.List[str]], + ) -> None: + + super().__init__(*args, **kwargs) + self._requires_any_args = requires_any_args or set() + self._requires_all_args = requires_all_args or set() + self._requires_any_patch = requires_any_patch or set() + self._requires_all_patch = requires_all_patch or set() + + @property + def requires_any_args(self) -> tp.Set[str]: + return self._requires_any_args + + @property + def requires_all_args(self) -> tp.Set[str]: + return self._requires_all_args + + @property + def requires_any_patch(self) -> tp.Set[str]: + return self._requires_any_patch + + @property + def requires_all_patch(self) -> tp.Set[str]: + return self._requires_all_patch + + def can_be_executed_by( + self, extra_args: tp.Set[str], + applied_patches: 'patch_provider.PatchSet' + ) -> bool: + """ + Checks whether this command can be executed with the give configuration. + + Args: + extra_args: additional command line arguments that will be passed to + the command + applied_patches: patches that were applied to create the executable + + Returns: + whether this command can be executed + """ + all_args = set(self._args).union(extra_args) + all_patch_tags: tp.Set[str] = set() + for patch in applied_patches: + if patch.feature_tags: + all_patch_tags.update(patch.feature_tags) + + return bool(( + not self.requires_any_args or + all_args.intersection(self.requires_any_args) + ) and ( + not self.requires_all_args or + self.requires_all_args.issubset(all_args) + ) and ( + not self.requires_any_patch or + all_patch_tags.intersection(self.requires_any_patch) + ) and ( + not self.requires_all_patch or + self.requires_all_patch.issubset(all_patch_tags) + )) diff --git a/varats-core/varats/provider/patch/patch_provider.py b/varats-core/varats/provider/patch/patch_provider.py index d466a629b..15aea0512 100644 --- a/varats-core/varats/provider/patch/patch_provider.py +++ b/varats-core/varats/provider/patch/patch_provider.py @@ -38,7 +38,8 @@ def __init__( description: str, path: Path, valid_revisions: tp.Optional[tp.Set[CommitHash]] = None, - tags: tp.Optional[tp.Set[str]] = None + tags: tp.Optional[tp.Set[str]] = None, + feature_tags: tp.Optional[tp.Set[str]] = None ): self.project_name: str = project_name self.shortname: str = shortname @@ -47,9 +48,10 @@ def __init__( self.valid_revisions: tp.Set[ CommitHash] = valid_revisions if valid_revisions else set() self.tags: tp.Optional[tp.Set[str]] = tags + self.feature_tags: tp.Optional[tp.Set[str]] = feature_tags @staticmethod - def from_yaml(yaml_path: Path): + def from_yaml(yaml_path: Path) -> 'Patch': """Creates a Patch from a YAML file.""" yaml_dict = yaml.safe_load(yaml_path.read_text()) @@ -62,16 +64,17 @@ def from_yaml(yaml_path: Path): # the yaml info file. path = yaml_path.parent / path - tags = None - if "tags" in yaml_dict: - tags = yaml_dict["tags"] + tags = yaml_dict.get("tags") + feature_tags = yaml_dict.get("feature_tags") project_git_path = get_local_project_git_path(project_name) # Update repository to have all upstream changes fetch_repository(project_git_path) - def parse_revisions(rev_dict: tp.Dict) -> tp.Set[CommitHash]: + def parse_revisions( + rev_dict: tp.Dict[str, tp.Any] + ) -> tp.Set[CommitHash]: res: tp.Set[CommitHash] = set() if "single_revision" in rev_dict: @@ -102,10 +105,11 @@ def parse_revisions(rev_dict: tp.Dict) -> tp.Set[CommitHash]: return res + include_revisions: tp.Set[CommitHash] if "include_revisions" in yaml_dict: include_revisions = parse_revisions(yaml_dict["include_revisions"]) else: - include_revisions: tp.Set[CommitHash] = set( + include_revisions = set( get_all_revisions_between( get_initial_commit(project_git_path).hash, "", ShortCommitHash, project_git_path @@ -118,7 +122,8 @@ def parse_revisions(rev_dict: tp.Dict) -> tp.Set[CommitHash]: ) return Patch( - project_name, shortname, description, path, include_revisions, tags + project_name, shortname, description, path, include_revisions, tags, + feature_tags ) def __repr__(self) -> str: @@ -137,18 +142,21 @@ def __str__(self) -> str: return str_representation - def __hash__(self): + def __hash__(self) -> int: + hash_args = [self.shortname, self.path] if self.tags: - return hash((self.shortname, str(self.path), tuple(self.tags))) + hash_args += tuple(self.tags) + if self.feature_tags: + hash_args += tuple(self.feature_tags) - return hash((self.shortname, str(self.path))) + return hash(tuple(hash_args)) class PatchSet: """A PatchSet is a storage container for project specific patches that can easily be accessed via the tags of a patch.""" - def __init__(self, patches: tp.Set[Patch]): + def __init__(self, patches: tp.Union[tp.Set[Patch], tp.FrozenSet[Patch]]): self.__patches: tp.FrozenSet[Patch] = frozenset(patches) def __iter__(self) -> tp.Iterator[Patch]: @@ -160,7 +168,7 @@ def __contains__(self, value: tp.Any) -> bool: def __len__(self) -> int: return len(self.__patches) - def __getitem__(self, tags: tp.Union[str, tp.Iterable[str]]): + def __getitem__(self, tags: tp.Union[str, tp.Iterable[str]]) -> 'PatchSet': """ Overrides the bracket operator of a PatchSet. @@ -213,6 +221,30 @@ def all_of(self, tags: tp.Union[str, tp.Iterable[str]]) -> "PatchSet": """ return self[tags] + def any_of_features(self, feature_tags: tp.Iterable[str]) -> "PatchSet": + """Returns a patch set with patches containing at least one of the given + feature tags.""" + tag_set = set(feature_tags) + result: tp.Set[Patch] = set() + for patch in self: + if patch.feature_tags and patch.feature_tags.intersection(tag_set): + result.add(patch) + + return PatchSet(result) + + def all_of_features( + self, feature_tags: tp.Union[str, tp.Iterable[str]] + ) -> "PatchSet": + """Returns a patch set with patches containing all the given feature + tags.""" + tag_set = set(feature_tags) + result: tp.Set[Patch] = set() + for patch in self: + if patch.feature_tags and tag_set.issubset(patch.feature_tags): + result.add(patch) + + return PatchSet(result) + def __hash__(self) -> int: return hash(self.__patches) @@ -286,7 +318,7 @@ def get_patches_for_revision(self, revision: CommitHash) -> PatchSet: @classmethod def create_provider_for_project( cls: tp.Type[ProviderType], project: tp.Type[Project] - ): + ) -> 'PatchProvider': """ Creates a provider instance for the given project. @@ -302,7 +334,7 @@ def create_provider_for_project( @classmethod def create_default_provider( cls: tp.Type[ProviderType], project: tp.Type[Project] - ): + ) -> 'PatchProvider': """ Creates a default provider instance that can be used with any project. @@ -315,10 +347,11 @@ def create_default_provider( @classmethod def _get_patches_repository_path(cls) -> Path: - return Path(target_prefix()) / cls.patches_source.local + # pathlib doesn't have type annotations for '/' + return tp.cast(Path, Path(target_prefix()) / cls.patches_source.local) @classmethod - def _update_local_patches_repo(cls): + def _update_local_patches_repo(cls) -> None: lock_path = Path(target_prefix()) / "patch_provider.lock" with lock_file(lock_path): diff --git a/varats-core/varats/utils/filesystem_util.py b/varats-core/varats/utils/filesystem_util.py index bc44c3265..258fb5e27 100644 --- a/varats-core/varats/utils/filesystem_util.py +++ b/varats-core/varats/utils/filesystem_util.py @@ -18,7 +18,8 @@ def __init__(self, folder: tp.Union[Path, str]) -> None: @contextmanager -def lock_file(lock_path: Path, lock_mode: int = fcntl.LOCK_EX) -> tp.Generator: +def lock_file(lock_path: Path, + lock_mode: int = fcntl.LOCK_EX) -> tp.Generator[None, None, None]: open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC lock_fd = os.open(lock_path, open_mode) try: diff --git a/varats/varats/projects/c_projects/xz.py b/varats/varats/projects/c_projects/xz.py index 3299a0e08..3d1a580ed 100644 --- a/varats/varats/projects/c_projects/xz.py +++ b/varats/varats/projects/c_projects/xz.py @@ -18,13 +18,13 @@ from varats.paper.paper_config import PaperConfigSpecificGit from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( - VCommand, ProjectBinaryWrapper, get_local_project_git_path, BinaryType, verify_binaries, ) from varats.project.sources import FeatureSource +from varats.project.varats_command import VCommand from varats.project.varats_project import VProject from varats.utils.git_util import ( ShortCommitHash, @@ -111,7 +111,7 @@ class Xz(VProject): output=SourceRoot("geo-maps/countries-land-250m.geo.json"), label="countries-land-250m", creates=["geo-maps/countries-land-250m.geo.json.xz"], - requires_all={"--compress"}, + requires_all_args={"--compress"}, ) ], } From db6a69b567a2dc5787b9fa44db466495ce74b243 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20B=C3=B6hm?= Date: Wed, 27 Sep 2023 12:00:35 +0200 Subject: [PATCH 8/8] Adds synthetic case sudies with different implementation patterns. (#840) --- .../SynthIPTemplate_0.case_study | 25 ++ tests/experiment/test_workload_util.py | 37 +++ tests/utils/test_experiment_util.py | 20 ++ .../perf_tests/feature_perf_cs_collection.py | 307 ++++++++++++++++++ 4 files changed, 389 insertions(+) create mode 100644 tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPTemplate_0.case_study diff --git a/tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPTemplate_0.case_study b/tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPTemplate_0.case_study new file mode 100644 index 000000000..28324d99a --- /dev/null +++ b/tests/TEST_INPUTS/paper_configs/test_config_ids/SynthIPTemplate_0.case_study @@ -0,0 +1,25 @@ +--- +DocType: CaseStudy +Version: 1 +... +--- +project_name: SynthIPTemplate +stages: +- revisions: + - commit_hash: 793035062810ea3a2d9a10f831cd199fbbb82090 + commit_id: 64 + config_ids: + - 0 + - 1 + - 2 + - 3 + - 4 +version: 0 +... +--- +config_type: PatchConfiguration +0: '["Decompress"]' +1: '["Compress"]' +2: '["Compress", "fastmode", "no_smallmode"]' +3: '["Compress", "no_fastmode", "smallmode"]' +4: '["Compress", "fastmode", "smallmode"]' diff --git a/tests/experiment/test_workload_util.py b/tests/experiment/test_workload_util.py index b8de4db27..e98e56d00 100644 --- a/tests/experiment/test_workload_util.py +++ b/tests/experiment/test_workload_util.py @@ -6,8 +6,14 @@ from benchbuild.source.base import Revision, Variant import varats.experiment.workload_util as wu +from tests.helper_utils import run_in_test_environment, UnitTestFixtures +from varats.paper.paper_config import load_paper_config from varats.projects.c_projects.xz import Xz +from varats.projects.perf_tests.feature_perf_cs_collection import ( + SynthIPTemplate, +) from varats.utils.git_util import ShortCommitHash +from varats.utils.settings import vara_cfg TT = PathToken.make_token(RootRenderer()) @@ -66,6 +72,37 @@ def test_workload_commands_requires(self) -> None: ) self.assertEqual(len(commands), 1) + @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS) + def test_workload_commands_requires_patch(self) -> None: + vara_cfg()['paper_config']['current_config'] = "test_config_ids" + load_paper_config() + + revision = Revision( + SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"), + Variant(SynthIPTemplate.SOURCE[1], "1") + ) + project = SynthIPTemplate(revision=revision) + binary = SynthIPTemplate.binaries_for_revision( + ShortCommitHash("7930350628") + )[0] + workloads = wu.workload_commands(project, binary, []) + self.assertEqual(len(workloads), 2) + + @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS) + def test_workload_commands_requires_patch2(self) -> None: + vara_cfg()['paper_config']['current_config'] = "test_config_ids" + load_paper_config() + + revision = Revision( + SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"), + Variant(SynthIPTemplate.SOURCE[1], "0") + ) + project = SynthIPTemplate(revision=revision) + binary = SynthIPTemplate \ + .binaries_for_revision(ShortCommitHash("7930350628"))[0] + workloads = wu.workload_commands(project, binary, []) + self.assertEqual(len(workloads), 0) + class TestWorkloadFilenames(unittest.TestCase): diff --git a/tests/utils/test_experiment_util.py b/tests/utils/test_experiment_util.py index 90061005b..a6bc93bd1 100644 --- a/tests/utils/test_experiment_util.py +++ b/tests/utils/test_experiment_util.py @@ -24,6 +24,9 @@ from varats.project.project_util import BinaryType, ProjectBinaryWrapper from varats.project.varats_project import VProject from varats.projects.c_projects.xz import Xz +from varats.projects.perf_tests.feature_perf_cs_collection import ( + SynthIPTemplate, +) from varats.report.gnu_time_report import TimeReport from varats.report.report import FileStatusExtension, ReportSpecification from varats.utils.git_util import ShortCommitHash @@ -419,3 +422,20 @@ def test_get_extra_config_options(self) -> None: ) project = Xz(revision=revision) self.assertEqual(EU.get_extra_config_options(project), ["--foo"]) + + @run_in_test_environment(UnitTestFixtures.PAPER_CONFIGS) + def test_get_config_patches(self) -> None: + vara_cfg()['paper_config']['current_config'] = "test_config_ids" + load_paper_config() + + revision = Revision( + SynthIPTemplate, Variant(SynthIPTemplate.SOURCE[0], "7930350628"), + Variant(SynthIPTemplate.SOURCE[1], "4") + ) + project = SynthIPTemplate(revision=revision) + patches = EU.get_config_patches(project) + self.assertEqual(len(patches), 1) + self.assertEqual( + list(patches)[0].feature_tags, + ["Compress", "fastmode", "smallmode"] + ) diff --git a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py index cea24265b..6ff8db619 100644 --- a/varats/varats/projects/perf_tests/feature_perf_cs_collection.py +++ b/varats/varats/projects/perf_tests/feature_perf_cs_collection.py @@ -4,6 +4,7 @@ import benchbuild as bb from benchbuild.command import Command, SourceRoot, WorkloadSet +from benchbuild.source import HTTPMultiple from benchbuild.utils.cmd import make, cmake, mkdir from benchbuild.utils.revision_ranges import RevisionRange from benchbuild.utils.settings import get_number_of_jobs @@ -19,6 +20,7 @@ verify_binaries, ) from varats.project.sources import FeatureSource +from varats.project.varats_command import VCommand from varats.project.varats_project import VProject from varats.utils.git_commands import init_all_submodules, update_all_submodules from varats.utils.git_util import RevisionBinaryMap, ShortCommitHash @@ -394,3 +396,308 @@ def compile(self) -> None: def recompile(self) -> None: """Recompile the project.""" _do_feature_perf_cs_collection_recompile(self) + + +class SynthIPRuntime(VProject): + """Synthetic case-study project for testing flow sensitivity.""" + + NAME = 'SynthIPRuntime' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local="SynthIPRuntime", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("SynthIPRuntime") + ), + HTTPMultiple( + local="geo-maps", + remote={ + "1.0": + "https://github.com/simonepri/geo-maps/releases/" + "download/v0.6.0" + }, + files=["countries-land-1km.geo.json", "countries-land-1m.geo.json"] + ), + FeatureSource() + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.SMALL): [ + VCommand( + SourceRoot("SynthIPRuntime") / RSBinary("Runtime"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1km", + creates=["geo-maps/countries-land-1km.geo.json.compressed"], + requires_all_args={"-c"} + ) + ], + WorkloadSet(WorkloadCategory.MEDIUM): [ + VCommand( + SourceRoot("SynthIPRuntime") / RSBinary("Runtime"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1m", + creates=["geo-maps/countries-land-1m.geo.json.compressed"], + requires_all_args={"-c"} + ) + ], + } + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + return RevisionBinaryMap( + get_local_project_git_path(SynthIPRuntime.NAME) + ).specify_binary( + "build/bin/Runtime", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("4151c42ffe", "master") + )[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_SYNTHIPRUNTIME" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self) + + +class SynthIPTemplate(VProject): + """Synthetic case-study project for testing flow sensitivity.""" + + NAME = 'SynthIPTemplate' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local="SynthIPTemplate", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("SynthIPTemplate") + ), + FeatureSource() + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.SMALL): [ + VCommand( + SourceRoot("SynthIPTemplate") / RSBinary("Template"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1km", + creates=["geo-maps/countries-land-1km.geo.json.compressed"], + requires_all_patch={"Compress"} + ) + ], + WorkloadSet(WorkloadCategory.MEDIUM): [ + VCommand( + SourceRoot("SynthIPTemplate") / RSBinary("Template"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1m", + creates=["geo-maps/countries-land-1m.geo.json.compressed"], + requires_all_patch={"Compress"} + ) + ], + } + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + return RevisionBinaryMap( + get_local_project_git_path(SynthIPTemplate.NAME) + ).specify_binary( + "build/bin/Template", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("4151c42ffe", "master") + )[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_SYNTHIPTEMPLATE" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self) + + +class SynthIPTemplate2(VProject): + """Synthetic case-study project for testing flow sensitivity.""" + + NAME = 'SynthIPTemplate2' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local="SynthIPTemplate2", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("SynthIPTemplate2") + ), + FeatureSource() + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.SMALL): [ + VCommand( + SourceRoot("SynthIPTemplate2") / RSBinary("Template2"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1km", + creates=["geo-maps/countries-land-1km.geo.json.compressed"], + requires_all_patch={"Compress"} + ) + ], + WorkloadSet(WorkloadCategory.MEDIUM): [ + VCommand( + SourceRoot("SynthIPTemplate2") / RSBinary("Template2"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1m", + creates=["geo-maps/countries-land-1m.geo.json.compressed"], + requires_all_patch={"Compress"} + ) + ], + } + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + return RevisionBinaryMap( + get_local_project_git_path(SynthIPTemplate2.NAME) + ).specify_binary( + "build/bin/Template2", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("4151c42ffe", "master") + )[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_SYNTHIPTEMPLATE2" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self) + + +class SynthIPCombined(VProject): + """Synthetic case-study project for testing flow sensitivity.""" + + NAME = 'SynthIPCombined' + GROUP = 'perf_tests' + DOMAIN = ProjectDomains.TEST + + SOURCE = [ + bb.source.Git( + remote="https://github.com/se-sic/FeaturePerfCSCollection.git", + local="SynthIPCombined", + refspec="origin/HEAD", + limit=None, + shallow=False, + version_filter=project_filter_generator("SynthIPCombined") + ), + FeatureSource() + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.SMALL): [ + VCommand( + SourceRoot("SynthIPCombined") / RSBinary("Combined"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1km", + creates=["geo-maps/countries-land-1km.geo.json.compressed"], + requires_all_args={"-c"} + ) + ], + WorkloadSet(WorkloadCategory.MEDIUM): [ + VCommand( + SourceRoot("SynthIPCombined") / RSBinary("Combined"), + "-c", + "<", + "geo-maps/countries-land-1km.geo.json", + ">", + "geo-maps/countries-land-1km.geo.json.compressed", + label="countries-land-1m", + creates=["geo-maps/countries-land-1m.geo.json.compressed"], + requires_all_args={"-c"} + ) + ], + } + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash # pylint: disable=W0613 + ) -> tp.List[ProjectBinaryWrapper]: + return RevisionBinaryMap( + get_local_project_git_path(SynthIPCombined.NAME) + ).specify_binary( + "build/bin/Combined", + BinaryType.EXECUTABLE, + only_valid_in=RevisionRange("4151c42ffe", "master") + )[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + _do_feature_perf_cs_collection_compile( + self, "FPCSC_ENABLE_PROJECT_SYNTHIPCOMBINED" + ) + + def recompile(self) -> None: + """Recompile the project.""" + _do_feature_perf_cs_collection_recompile(self)