diff --git a/docs/source/research_tool_docs/vara/add_setup_infos/clion_setup.rst b/docs/source/research_tool_docs/vara/add_setup_infos/clion_setup.rst index ae03bf107..98a5c6869 100644 --- a/docs/source/research_tool_docs/vara/add_setup_infos/clion_setup.rst +++ b/docs/source/research_tool_docs/vara/add_setup_infos/clion_setup.rst @@ -22,24 +22,31 @@ How to set up VaRA/LLVM in CLion -DBUILD_SHARED_LIBS=ON -DCMAKE_C_FLAGS_DEBUG="-O2 -g -fno-omit-frame-pointer" -DCMAKE_CXX_FLAGS_DEBUG="-O2 -g -fno-omit-frame-pointer" + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,--undefined-version" -DCMAKE_CXX_STANDARD=17 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_INSTALL_PREFIX=/tools/VaRA -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_ENABLE_EH=ON - -DLLVM_ENABLE_LDD=ON + -DLLVM_ENABLE_LLD=ON -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;clang-tools-extra;vara;phasar" -DLLVM_ENABLE_RTTI=ON -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_PARALLEL_LINK_JOBS=4 -DLLVM_PHASAR_BUILD=ON + -DLLVM_TOOL_PHASAR_BUILD=ON + -DPHASAR_ENABLE_DYNAMIC_LOG=OFF + -DPHASAR_BUILD_IR=OFF + -DPHASAR_BUILD_UNITTESTS=OFF -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_TOOL_PHASAR_BUILD=ON - -DLLVM_USE_NEWPM=ON -DUSE_HTTPS=OFF -DUSE_SSH=OFF -DVARA_BUILD_LIBGIT=ON + -DVARA_FEATURE_BUILD_PYTHON_BINDINGS=OFF + -DVARA_FEATURE_BUILD_Z3_SOLVER=ON + -DVARA_FEATURE_USE_Z3_SOLVER=ON Use ``-O0`` for debug builds and ``-O2`` for development builds. @@ -57,22 +64,31 @@ How to set up VaRA/LLVM in CLion -DBUILD_SHARED_LIBS=ON -DCMAKE_C_FLAGS_RELEASE="-O3 -DNDEBUG -march=native -fno-omit-frame-pointer -gmlt" -DCMAKE_CXX_FLAGS_RELEASE="-O3 -DNDEBUG -march=native -fno-omit-frame-pointer -gmlt" + -DCMAKE_SHARED_LINKER_FLAGS="-Wl,--undefined-version" -DCMAKE_CXX_STANDARD=17 + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_INSTALL_PREFIX=/tools/VaRA -DLLVM_ENABLE_ASSERTIONS=OFF -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_ENABLE_EH=ON - -DLLVM_ENABLE_LDD=ON + -DLLVM_ENABLE_LLD=ON -DLLVM_ENABLE_PROJECTS="clang;lld;compiler-rt;clang-tools-extra;vara;phasar" -DLLVM_ENABLE_RTTI=ON + -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_PARALLEL_LINK_JOBS=4 -DLLVM_PHASAR_BUILD=ON + -DLLVM_TOOL_PHASAR_BUILD=ON + -DPHASAR_ENABLE_DYNAMIC_LOG=OFF + -DPHASAR_BUILD_IR=OFF + -DPHASAR_BUILD_UNITTESTS=OFF -DLLVM_TARGETS_TO_BUILD=X86 -DLLVM_TOOL_PHASAR_BUILD=ON - -DLLVM_USE_NEWPM=ON -DUSE_HTTPS=OFF -DUSE_SSH=OFF -DVARA_BUILD_LIBGIT=ON + -DVARA_FEATURE_BUILD_PYTHON_BINDINGS=OFF + -DVARA_FEATURE_BUILD_Z3_SOLVER=ON + -DVARA_FEATURE_USE_Z3_SOLVER=ON - **Build directory:** ``/tools_src/vara-llvm-project/build/dev-clion`` - **Build options:** leave empty diff --git a/tests/utils/test_filesystem_util.py b/tests/utils/test_filesystem_util.py new file mode 100644 index 000000000..4e444976a --- /dev/null +++ b/tests/utils/test_filesystem_util.py @@ -0,0 +1,48 @@ +"""Test the filesystem utils of VaRA-TS.""" +import errno +import os +import unittest +import uuid +from fcntl import flock, LOCK_EX, LOCK_NB, LOCK_UN + +from varats.utils.filesystem_util import lock_file + + +class TestFileLock(unittest.TestCase): + """Tests whether the lock context manager works correctly.""" + + def test_file_locking(self): + """Test that the file is locked when in a context manager.""" + tmp_lock_file = "/tmp/lock-test.lock" + + with lock_file(tmp_lock_file): + # File should automatically be created + self.assertTrue(os.path.exists(tmp_lock_file)) + + f = os.open(tmp_lock_file, os.O_RDONLY) + + with self.assertRaises(OSError) as context: + # A non-blocking attempt to lock the file again should fail immediately + flock(f, LOCK_EX | LOCK_NB) + os.close(f) + self.assertEqual(context.exception.errno, errno.EWOULDBLOCK) + + # Attempting to lock the file and immediately unlocking should now work + f = os.open(tmp_lock_file, os.O_RDONLY) + flock(f, LOCK_EX | LOCK_NB) + flock(f, LOCK_UN) + os.close(f) + + def test_lock_file_new_folder(self): + """Test that the lock context manager works correctly when the lock file + is in a new folder.""" + tmp_lock_file = f"/tmp/{uuid.uuid4()}" + + while os.path.isdir(tmp_lock_file): + tmp_lock_file = f"/tmp/{uuid.uuid4()}" + + tmp_lock_file += "/lock-test.lock" + + with lock_file(tmp_lock_file): + # File should automatically be created + self.assertTrue(os.path.exists(tmp_lock_file)) diff --git a/varats-core/varats/provider/feature/feature_model_provider.py b/varats-core/varats/provider/feature/feature_model_provider.py index 90c9bf4d8..69d2d7cca 100644 --- a/varats-core/varats/provider/feature/feature_model_provider.py +++ b/varats-core/varats/provider/feature/feature_model_provider.py @@ -7,6 +7,7 @@ from benchbuild.source.base import target_prefix from varats.provider.provider import Provider +from varats.utils.filesystem_util import lock_file class FeatureModelNotFound(FileNotFoundError): @@ -90,6 +91,9 @@ def _get_feature_model_repository_path() -> Path: refspec="origin/HEAD", limit=1, ) - fm_source.fetch() + + lock_path = Path(target_prefix()) / "fm_provider.lock" + with lock_file(lock_path): + fm_source.fetch() return Path(Path(target_prefix()) / fm_source.local) diff --git a/varats-core/varats/utils/filesystem_util.py b/varats-core/varats/utils/filesystem_util.py index 258fb5e27..4f6ff107d 100644 --- a/varats-core/varats/utils/filesystem_util.py +++ b/varats-core/varats/utils/filesystem_util.py @@ -20,6 +20,9 @@ def __init__(self, folder: tp.Union[Path, str]) -> None: @contextmanager def lock_file(lock_path: Path, lock_mode: int = fcntl.LOCK_EX) -> tp.Generator[None, None, None]: + # Create directories until lock file if required + os.makedirs(os.path.dirname(lock_path), exist_ok=True) + open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC lock_fd = os.open(lock_path, open_mode) try: diff --git a/varats/varats/experiments/vara/feature_perf_precision.py b/varats/varats/experiments/vara/feature_perf_precision.py index 2c58fcee4..185931615 100644 --- a/varats/varats/experiments/vara/feature_perf_precision.py +++ b/varats/varats/experiments/vara/feature_perf_precision.py @@ -16,6 +16,7 @@ from plumbum import local, BG from plumbum.commands.modifiers import Future +from varats.base.configuration import PatchConfiguration from varats.data.reports.performance_influence_trace_report import ( PerfInfluenceTraceReportAggregate, ) @@ -44,7 +45,7 @@ from varats.report.report import ReportSpecification from varats.report.tef_report import TEFReportAggregate from varats.tools.research_tools.vara import VaRA -from varats.utils.config import get_current_config_id +from varats.utils.config import get_current_config_id, get_config from varats.utils.git_util import ShortCommitHash REPS = 3 @@ -77,10 +78,24 @@ def perf_prec_workload_commands( def select_project_binaries(project: VProject) -> tp.List[ProjectBinaryWrapper]: """Uniformly select the binaries that should be analyzed.""" if project.name == "DunePerfRegression": - return [ - binary for binary in project.binaries - if binary.name == "poisson_yasp_q2_3d" - ] + config = get_config(project, PatchConfiguration) + if not config: + return [] + + f_tags = {opt.value for opt in config.options()} + + grid_binary_map = { + "YaspGrid": "poisson_yasp_q2_3d", + "UGGrid": "poisson_ug_pk_2d", + "ALUGrid": "poisson_alugrid" + } + + for grid, binary_name in grid_binary_map.items(): + if grid in f_tags: + return [ + binary for binary in project.binaries + if binary.name == binary_name + ] return [project.binaries[0]] @@ -101,12 +116,12 @@ def get_threshold(project: VProject) -> int: "SynthSAFieldSensitivity", "SynthIPRuntime", "SynthIPTemplate", "SynthIPTemplate2", "SynthIPCombined" ]: - # Don't instrument everything for these synthtic projects + # Don't instrument everything for these synthetic projects return 10 return 0 - if project.name in ["HyTeg"]: + if project.name in ["HyTeg", "PicoSATLoadTime"]: return 0 return 100 @@ -343,7 +358,7 @@ def run_traced_code(self, tmp_dir: Path) -> StepResult: f"Running example {prj_command.command.label}" ) - bpf_runner = bpf_runner = self.attach_usdt_bcc( + bpf_runner = self.attach_usdt_bcc( local_tracefile_path, self.project.source_of_primary / self._binary.path diff --git a/varats/varats/plots/feature_perf_precision.py b/varats/varats/plots/feature_perf_precision.py index eee2a25cc..6b232b1c2 100644 --- a/varats/varats/plots/feature_perf_precision.py +++ b/varats/varats/plots/feature_perf_precision.py @@ -22,6 +22,7 @@ from varats.plot.plot import Plot from varats.plot.plots import PlotGenerator from varats.plots.scatter_plot_utils import multivariate_grid +from varats.ts_utils.click_param_types import REQUIRE_MULTI_CASE_STUDY from varats.utils.exceptions import UnsupportedOperation from varats.utils.git_util import FullCommitHash @@ -79,7 +80,7 @@ class PerfPrecisionDistPlot(Plot, plot_name='fperf_precision_dist'): different profilers.""" def plot(self, view_mode: bool) -> None: - case_studies = get_loaded_paper_config().get_all_case_studies() + case_studies = self.plot_kwargs["case_studies"] profilers: tp.List[Profiler] = [VXray(), PIMTracer(), EbpfTraceTEF()] # Data aggregation @@ -153,12 +154,40 @@ def calc_missing_revisions( class PerfProfDistPlotGenerator( - PlotGenerator, generator_name="fperf-precision-dist", options=[] + PlotGenerator, + generator_name="fperf-precision-dist", + options=[REQUIRE_MULTI_CASE_STUDY] ): - """Generates performance distribution plot.""" + """Generates performance distribution plot for a given list of case + studies.""" def generate(self) -> tp.List[Plot]: - return [PerfPrecisionDistPlot(self.plot_config, **self.plot_kwargs)] + case_studies = self.plot_kwargs.pop("case_study") + return [ + PerfPrecisionDistPlot( + self.plot_config, case_studies=case_studies, **self.plot_kwargs + ) + ] + + +class PerfProfDistPlotGeneratorForEachCS( + PlotGenerator, + generator_name="fperf-precision-dist-cs", + options=[REQUIRE_MULTI_CASE_STUDY] +): + """Generates performance distribution plot for each of the given case + studies.""" + + def generate(self) -> tp.List[Plot]: + case_studies = self.plot_kwargs.pop("case_study") + return [ + PerfPrecisionDistPlot( + self.plot_config, + case_study=case_study, + case_studies=[case_study], + **self.plot_kwargs + ) for case_study in case_studies + ] class PerfOverheadPlot(Plot, plot_name='fperf_overhead'): diff --git a/varats/varats/projects/c_projects/picosat.py b/varats/varats/projects/c_projects/picosat.py index 71de307e6..569f38a2f 100644 --- a/varats/varats/projects/c_projects/picosat.py +++ b/varats/varats/projects/c_projects/picosat.py @@ -3,14 +3,18 @@ import typing as tp import benchbuild as bb -from benchbuild.command import WorkloadSet, Command, SourceRoot +from benchbuild.command import WorkloadSet, SourceRoot from benchbuild.source import HTTP from benchbuild.source.http import HTTPUntar from benchbuild.utils.cmd import make from benchbuild.utils.settings import get_number_of_jobs from plumbum import local -from varats.experiment.workload_util import RSBinary, WorkloadCategory +from varats.experiment.workload_util import ( + RSBinary, + WorkloadCategory, + ConfigParams, +) from varats.paper.paper_config import PaperConfigSpecificGit from varats.project.project_domain import ProjectDomains from varats.project.project_util import ( @@ -21,6 +25,7 @@ verify_binaries, ) from varats.project.sources import FeatureSource +from varats.project.varats_command import VCommand from varats.project.varats_project import VProject from varats.provider.release.release_provider import ( ReleaseProviderHook, @@ -98,39 +103,39 @@ class PicoSAT(VProject, ReleaseProviderHook): WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ - Command( + VCommand( SourceRoot("picosat") / RSBinary("picosat"), "example.cnf", label="example.cnf", ) ], WorkloadSet(WorkloadCategory.SMALL): [ - Command( + VCommand( SourceRoot("picosat") / RSBinary("picosat"), "aim-100-1_6-no-1.cnf", label="aim-100-1-6-no-1.cnf", ) ], WorkloadSet(WorkloadCategory.MEDIUM): [ - Command( + VCommand( SourceRoot("picosat") / RSBinary("picosat"), "traffic_kkb_unknown.cnf/traffic_kkb_unknown.cnf", label="traffic-kkb-unknow.cnf", ), - Command( + VCommand( SourceRoot("picosat") / RSBinary("picosat"), "abw-N-bcsstk07.mtx-w44.cnf/abw-N-bcsstk07.mtx-w44.cnf", label="abw-N-bcsstk07.mtx-w44.cnf", ), ], WorkloadSet(WorkloadCategory.LARGE): [ - Command( + VCommand( SourceRoot("picosat") / RSBinary("picosat"), "UNSAT_H_instances_childsnack_p11.hddl_1.cnf/" "UNSAT_H_instances_childsnack_p11.hddl_1.cnf", label="UNSAT-H-instances-childsnack-p11.hddl-1.cnf", ), - Command( + VCommand( SourceRoot("picosat") / RSBinary("picosat"), "UNSAT_H_instances_childsnack_p12.hddl_1.cnf/" "UNSAT_H_instances_childsnack_p12.hddl_1.cnf", @@ -189,3 +194,152 @@ def get_release_revisions( return [(FullCommitHash(h), tag) for h, tag in tagged_commits if re.match(release_regex, tag)] + + +class PicoSATLoadTime(VProject, ReleaseProviderHook): + """Adapted version of picoSAT that has been refactored, such that it does + not require a field-sensitive analysis.""" + + NAME = 'PicoSATLoadTime' + GROUP = 'c_projects' + DOMAIN = ProjectDomains.SOLVER + + SOURCE = [ + PaperConfigSpecificGit( + project_name="PicoSATLoadTime", + remote="https://github.com/se-sic/picoSAT-vara", + local="PicoSATLoadTime", + refspec="origin/HEAD", + limit=None, + shallow=False + ), + FeatureSource(), + HTTP( + local="example.cnf", + remote={ + "1.0": + "https://github.com/se-sic/picoSAT-mirror/releases/" + "download/picoSAT-965/example.cnf" + } + ), + HTTP( + local="ibm-2004-03-k70.cnf", + remote={ + "1.0": + "https://github.com/se-sic/picoSAT-vara/releases/" + "download/workloads-sat-race-2006/ibm-2004-03-k70.cnf" + } + ), + HTTPUntar( + local="abw-N-bcsstk07.mtx-w44.cnf", + remote={ + "1.0": + "https://github.com/se-sic/picoSAT-mirror/releases/" + "download/picoSAT-965/abw-N-bcsstk07.mtx-w44.cnf.tar.gz" + } + ), + HTTPUntar( + local="traffic_kkb_unknown.cnf", + remote={ + "1.0": + "https://github.com/se-sic/picoSAT-mirror/releases/" + "download/picoSAT-965/traffic_kkb_unknown.cnf.tar.gz" + } + ), + HTTPUntar( + local="UNSAT_H_instances_childsnack_p11.hddl_1.cnf", + remote={ + "1.0": + "https://github.com/se-sic/picoSAT-mirror/releases/" + "download/picoSAT-965/" + "UNSAT_H_instances_childsnack_p11.hddl_1.cnf.tar.gz" + } + ), + HTTPUntar( + local="UNSAT_H_instances_childsnack_p12.hddl_1.cnf", + remote={ + "1.0": + "https://github.com/se-sic/picoSAT-mirror/releases/" + "download/picoSAT-965/" + "UNSAT_H_instances_childsnack_p12.hddl_1.cnf.tar.gz" + } + ), + ] + + WORKLOADS = { + WorkloadSet(WorkloadCategory.SMALL): [ + VCommand( + SourceRoot("PicoSATLoadTime") / RSBinary("picosat"), + "ibm-2004-03-k70.cnf", + ConfigParams(), + label="ibm-2004-03-k70.cnf", + ), + ], + } + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash + ) -> tp.List[ProjectBinaryWrapper]: + binary_map = RevisionBinaryMap( + get_local_project_git_path(PicoSATLoadTime.NAME) + ) + binary_map.specify_binary( + 'picosat', BinaryType.EXECUTABLE, valid_exit_codes=[0, 10, 20] + ) + + return binary_map[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + picosat_source = local.path(self.source_of(self.primary_source)) + + c_compiler = bb.compiler.cc(self) + cxx_compiler = bb.compiler.cxx(self) + + with local.cwd(picosat_source): + revisions_with_new_config_name = get_all_revisions_between( + "33c685e82213228726364980814f0183e435de78", "", ShortCommitHash + ) + picosat_version = ShortCommitHash(self.version_of_primary) + if picosat_version in revisions_with_new_config_name: + config_script_name = "./configure.sh" + else: + config_script_name = "./configure" + + with local.cwd(picosat_source): + with local.env(CC=str(c_compiler), CXX=str(cxx_compiler)): + bb.watch(local[config_script_name] + )(["--trace", "--stats", "-g"]) + bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) + + with local.cwd(picosat_source): + verify_binaries(self) + + def recompile(self) -> None: + """Re-Compile the project.""" + picosat_source = local.path(self.source_of(self.primary_source)) + c_compiler = bb.compiler.cc(self) + cxx_compiler = bb.compiler.cxx(self) + + with local.cwd(picosat_source): + with local.env(CC=str(c_compiler), CXX=str(cxx_compiler)): + bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) + + with local.cwd(picosat_source): + verify_binaries(self) + + @classmethod + def get_release_revisions( + cls, release_type: ReleaseType + ) -> tp.List[tp.Tuple[FullCommitHash, str]]: + release_regex = "^picoSAT-[0-9]+$" + + tagged_commits = get_tagged_commits(cls.NAME) + + return [(FullCommitHash(h), tag) + for h, tag in tagged_commits + if re.match(release_regex, tag)] diff --git a/varats/varats/tables/feature_perf_precision.py b/varats/varats/tables/feature_perf_precision.py index b11c6baa6..d1b2b4453 100644 --- a/varats/varats/tables/feature_perf_precision.py +++ b/varats/varats/tables/feature_perf_precision.py @@ -35,6 +35,33 @@ from varats.table.tables import TableFormat, TableGenerator from varats.utils.git_util import calc_repo_loc, ChurnConfig +GROUP_SYNTHETIC_CATEGORIES = True + +SYNTH_CATEGORIES = [ + "Static Analysis", "Dynamic Analysis", "Configurability", + "Implementation Pattern" +] + + +def compute_cs_category_grouping(case_study_name: str) -> str: + """Mapping function to transform individual project names to their synthtic + categories.""" + if case_study_name.startswith("SynthSA"): + return "Static Analysis" + + if case_study_name.startswith("SynthDA" + ) or case_study_name.startswith("SynthOV"): + return "Dynamic Analysis" + + if case_study_name.startswith("SynthFeature"): + return "Configurability" + + if case_study_name.startswith("SynthCT" + ) or case_study_name.startswith("SynthIP"): + return "Implementation Pattern" + + return case_study_name + def cmap_map( function: tp.Callable[[npt.NDArray[np.float64]], npt.NDArray[np.float64]], @@ -142,7 +169,7 @@ def _prepare_data_table( def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: """Setup performance precision table.""" case_studies = get_loaded_paper_config().get_all_case_studies() - profilers: tp.List[Profiler] = [VXray(), PIMTracer()] + profilers: tp.List[Profiler] = [VXray(), PIMTracer(), EbpfTraceTEF()] # Data aggregation df = self._prepare_data_table(case_studies, profilers) @@ -304,6 +331,21 @@ def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: precision_df, overhead_df, on=["CaseStudy", "Profiler"] ) + if GROUP_SYNTHETIC_CATEGORIES: + + merged_df["CaseStudy"] = merged_df["CaseStudy"].apply( + compute_cs_category_grouping + ) + merged_df = merged_df.groupby(['CaseStudy', "Profiler"], + as_index=False).agg({ + 'precision': 'mean', + 'recall': 'mean', + 'overhead_time': 'mean', + 'overhead_time_rel': 'mean', + 'overhead_memory_rel': 'mean', + 'overhead_memory': 'mean' + }) + pivot_df = merged_df.pivot( index='CaseStudy', columns='Profiler', @@ -324,7 +366,18 @@ def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: ], axis=1) - pivot_df.loc["Total"] = pivot_df.mean() + # All means need to be computed before they are added as rows + overall_mean = pivot_df.mean() + if GROUP_SYNTHETIC_CATEGORIES: + synth_mean = pivot_df.loc[pivot_df.index.isin(SYNTH_CATEGORIES) + ].mean() + real_world_mean = pivot_df.loc[~pivot_df.index. + isin(SYNTH_CATEGORIES)].mean() + + pivot_df.loc["SynthMean"] = synth_mean + pivot_df.loc["RealWorldMean"] = real_world_mean + + pivot_df.loc["OverallMean"] = overall_mean # Rename columns # pylint: disable=anomalous-backslash-in-string @@ -516,6 +569,16 @@ def tabulate(self, table_format: TableFormat, wrap_table: bool) -> str: cs_data.append(pd.DataFrame.from_dict(cs_dict, orient='index')) df = pd.concat(cs_data).sort_index() + df.index.name = 'CaseStudy' + + if GROUP_SYNTHETIC_CATEGORIES: + df.index = df.index.map(compute_cs_category_grouping) + + df = df.groupby(df.index.name, as_index=True).agg({ + 'NumConfig': 'sum', + 'Locs': 'sum', + 'Regressions': 'sum' + }) style = df.style kwargs: tp.Dict[str, tp.Any] = {}