From 1e79e51c36f7e869f05df1e28caf29bec7249ae3 Mon Sep 17 00:00:00 2001 From: Johannes Weissmann Date: Mon, 18 Mar 2024 13:38:25 +0100 Subject: [PATCH 1/9] Adds a BC file extension for Architecture, a bare-bones MDReport and ArchitectureAnalysis experiment --- varats-core/varats/experiment/wllvm.py | 1 + varats/varats/data/reports/md_report.py | 11 ++ .../vara/architecture_report_experiment.py | 112 ++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 varats/varats/data/reports/md_report.py create mode 100644 varats/varats/experiments/vara/architecture_report_experiment.py diff --git a/varats-core/varats/experiment/wllvm.py b/varats-core/varats/experiment/wllvm.py index 6321091c1..cd0dcc813 100644 --- a/varats-core/varats/experiment/wllvm.py +++ b/varats-core/varats/experiment/wllvm.py @@ -48,6 +48,7 @@ class BCFileExtensions(Enum): TBAA = "TBAA" FEATURE = 'feature' BLAME = "blame" + ARCH = "fvara-arch" def __lt__(self, other: tp.Any) -> bool: if isinstance(other, BCFileExtensions): diff --git a/varats/varats/data/reports/md_report.py b/varats/varats/data/reports/md_report.py new file mode 100644 index 000000000..5a047af17 --- /dev/null +++ b/varats/varats/data/reports/md_report.py @@ -0,0 +1,11 @@ +"""Metadata report implementation.""" + +from varats.report.report import BaseReport + + +class MDReport(BaseReport, shorthand="MD", file_type="txt"): + """ + An Metadata report which serves as a base class for the Architecture report. + """ + + # TODO Implement the MDReport class. diff --git a/varats/varats/experiments/vara/architecture_report_experiment.py b/varats/varats/experiments/vara/architecture_report_experiment.py new file mode 100644 index 000000000..a72647e41 --- /dev/null +++ b/varats/varats/experiments/vara/architecture_report_experiment.py @@ -0,0 +1,112 @@ +"""Implements an empty experiment that just compiles the project.""" + +import typing as tp + +from benchbuild import Project +from benchbuild.extensions import compiler, run, time +from benchbuild.utils import actions +from benchbuild.utils.cmd import touch + +from varats.data.reports.md_report import MDReport +from varats.experiment.experiment_util import ( + VersionExperiment, + ExperimentHandle, + exec_func_with_pe_error_handler, + get_default_compile_error_wrapped, + create_default_analysis_failure_handler, + create_new_success_result_filepath, +) +from varats.experiment.wllvm import RunWLLVM +from varats.project.varats_project import VProject +from varats.report.report import ReportSpecification +from varats.utils.config import get_current_config_id + + +class ArchitectureAnalysis(actions.ProjectStep): # type: ignore + """Analyses a project's architecture with VaRA and generates a report.""" + + NAME = "ArchitectureAnalysis" + DESCRIPTION = "Analyses the bitcode with -vara-arch of VaRA." + + project: VProject + + def __init__(self, project: Project, experiment_handle: ExperimentHandle): + super().__init__(project=project) + self.__experiment_handle = experiment_handle + + def __call__(self) -> actions.StepResult: + return self.analyze() + + def analyze(self) -> actions.StepResult: + # + + config_id = get_current_config_id(self.project) + + for binary in self.project.binaries: + result_file = create_new_success_result_filepath( + self.__experiment_handle, + MDReport, + self.project, binary, + config_id + ) + + opt_params = [ + "-O0", "-g0", "-fvara-GB", + "-S", "-fvara-arch", + f"-vara-report-outfile={result_file}", + get_cached_bc_file_path( + self.project, binary, [ + BCFileExtensions.NO_OPT, + BCFileExtensions.ARCH + ] + ) + ] + + run_cmd = opt[opt_params] + + exec_func_with_pe_error_handler( + run_cmd, + create_default_analysis_failure_handler( + self.__experiment_handle, self.project, MDReport + ) + ) + + return actions.StepResult.OK + + +# Please take care when changing this file, see docs experiments/just_compile +class ArchitectureReport(VersionExperiment, shorthand="ARE"): + """Generates an Architecture report file.""" + + NAME = "GenerateArchitectureReport" + + REPORT_SPEC = ReportSpecification(MDReport) + + def actions_for_project( + self, project: Project + ) -> tp.MutableSequence[actions.Step]: + """Returns the specified steps to run the project(s) specified in the + call in a fixed order.""" + + # Add the required runtime extensions to the project(s). + project.runtime_extension = run.RuntimeExtension(project, self) \ + << time.RunWithTime() + + # Add the required compiler extensions to the project(s). + project.compiler_extension = compiler.RunCompiler(project, self) \ + << RunWLLVM() \ + << run.WithTimeout() + + project.compile = get_default_compile_error_wrapped( + self.get_handle(), project, self.REPORT_SPEC.main_report + ) + + analysis_actions = [] + analysis_actions.append(actions.Compile(project)) + analysis_actions.append(EmptyAnalysis(project, self.get_handle())) + analysis_actions.append(actions.Clean(project)) + + return analysis_actions + + + From 2e3571ec0740ae1c4234c46cc35ce059ff54173f Mon Sep 17 00:00:00 2001 From: Johannes Weissmann Date: Mon, 18 Mar 2024 13:49:33 +0100 Subject: [PATCH 2/9] Adds "SynthSAWholeProgram" from "FeaturePerfCSCollection" as a project --- .../cpp_projects/featureperfcscollection.py | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 varats/varats/projects/cpp_projects/featureperfcscollection.py diff --git a/varats/varats/projects/cpp_projects/featureperfcscollection.py b/varats/varats/projects/cpp_projects/featureperfcscollection.py new file mode 100644 index 000000000..e8dd90854 --- /dev/null +++ b/varats/varats/projects/cpp_projects/featureperfcscollection.py @@ -0,0 +1,71 @@ +"""Based on the project file for opencv.""" +import typing as tp + +import benchbuild as bb +from benchbuild.utils.cmd import make, cmake +from benchbuild.utils.settings import get_number_of_jobs +from plumbum import local + +from varats.containers.containers import get_base_image, ImageBase +from varats.paper.paper_config import PaperConfigSpecificGit +from varats.project.project_domain import ProjectDomains +from varats.project.project_util import ( + ProjectBinaryWrapper, + BinaryType, + get_local_project_git_path, + verify_binaries, +) +from varats.project.varats_project import VProject +from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap +from varats.utils.settings import bb_cfg + + +class OpenCV(VProject): + """Open Source Computer Vision Library.""" + + NAME = 'featureperfcscollection' + GROUP = 'cpp_projects' + DOMAIN = ProjectDomains.CPP_LIBRARY + + SOURCE = [ + PaperConfigSpecificGit( + project_name="featureperfcscollection", + remote="git@github.com:se-sic/FeaturePerfCSCollection.git", + local="featureperfcscollection", + refspec="origin/HEAD", + limit=None, + shallow=False + ) + ] + + @staticmethod + def binaries_for_revision( + revision: ShortCommitHash + ) -> tp.List[ProjectBinaryWrapper]: + binary_map = RevisionBinaryMap(get_local_project_git_path(OpenCV.NAME)) + + # TODO: please add correct binary names + binary_map.specify_binary("MISSING", BinaryType.EXECUTABLE) + + return binary_map[revision] + + def run_tests(self) -> None: + pass + + def compile(self) -> None: + """Compile the project.""" + opencv_source = local.path(self.source_of(self.primary_source)) + + c_compiler = bb.compiler.cc(self) + cxx_compiler = bb.compiler.cxx(self) + build_folder = opencv_source / "build" + build_folder.mkdir() + with local.cwd(build_folder): + + with local.env(CC=str(c_compiler), CXX=str(cxx_compiler)): + bb.watch(cmake)("../projects/SynthSAWholeProgram") + + bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) + + with local.cwd(opencv_source): + verify_binaries(self) From d6b90a1bd91f040d4876514a62bc1deea8230a0e Mon Sep 17 00:00:00 2001 From: Johannes Weissmann Date: Wed, 20 Mar 2024 21:11:40 +0100 Subject: [PATCH 3/9] Revert "Adds 'SynthSAWholeProgram' from 'FeaturePerfCSCollection' as a project" This reverts commit 2e3571ec0740ae1c4234c46cc35ce059ff54173f. --- .../cpp_projects/featureperfcscollection.py | 71 ------------------- 1 file changed, 71 deletions(-) delete mode 100644 varats/varats/projects/cpp_projects/featureperfcscollection.py diff --git a/varats/varats/projects/cpp_projects/featureperfcscollection.py b/varats/varats/projects/cpp_projects/featureperfcscollection.py deleted file mode 100644 index e8dd90854..000000000 --- a/varats/varats/projects/cpp_projects/featureperfcscollection.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Based on the project file for opencv.""" -import typing as tp - -import benchbuild as bb -from benchbuild.utils.cmd import make, cmake -from benchbuild.utils.settings import get_number_of_jobs -from plumbum import local - -from varats.containers.containers import get_base_image, ImageBase -from varats.paper.paper_config import PaperConfigSpecificGit -from varats.project.project_domain import ProjectDomains -from varats.project.project_util import ( - ProjectBinaryWrapper, - BinaryType, - get_local_project_git_path, - verify_binaries, -) -from varats.project.varats_project import VProject -from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap -from varats.utils.settings import bb_cfg - - -class OpenCV(VProject): - """Open Source Computer Vision Library.""" - - NAME = 'featureperfcscollection' - GROUP = 'cpp_projects' - DOMAIN = ProjectDomains.CPP_LIBRARY - - SOURCE = [ - PaperConfigSpecificGit( - project_name="featureperfcscollection", - remote="git@github.com:se-sic/FeaturePerfCSCollection.git", - local="featureperfcscollection", - refspec="origin/HEAD", - limit=None, - shallow=False - ) - ] - - @staticmethod - def binaries_for_revision( - revision: ShortCommitHash - ) -> tp.List[ProjectBinaryWrapper]: - binary_map = RevisionBinaryMap(get_local_project_git_path(OpenCV.NAME)) - - # TODO: please add correct binary names - binary_map.specify_binary("MISSING", BinaryType.EXECUTABLE) - - return binary_map[revision] - - def run_tests(self) -> None: - pass - - def compile(self) -> None: - """Compile the project.""" - opencv_source = local.path(self.source_of(self.primary_source)) - - c_compiler = bb.compiler.cc(self) - cxx_compiler = bb.compiler.cxx(self) - build_folder = opencv_source / "build" - build_folder.mkdir() - with local.cwd(build_folder): - - with local.env(CC=str(c_compiler), CXX=str(cxx_compiler)): - bb.watch(cmake)("../projects/SynthSAWholeProgram") - - bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) - - with local.cwd(opencv_source): - verify_binaries(self) From ca641e2d71da213de72df65eee78ebd60ee7d0ca Mon Sep 17 00:00:00 2001 From: Johannes Weissmann Date: Wed, 20 Mar 2024 21:21:38 +0100 Subject: [PATCH 4/9] Adds a proper BC file extension for Architecture --- varats-core/varats/experiment/wllvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varats-core/varats/experiment/wllvm.py b/varats-core/varats/experiment/wllvm.py index cd0dcc813..75b196841 100644 --- a/varats-core/varats/experiment/wllvm.py +++ b/varats-core/varats/experiment/wllvm.py @@ -48,7 +48,7 @@ class BCFileExtensions(Enum): TBAA = "TBAA" FEATURE = 'feature' BLAME = "blame" - ARCH = "fvara-arch" + ARCH = "architecture" def __lt__(self, other: tp.Any) -> bool: if isinstance(other, BCFileExtensions): From 6fe417a790cfe9bdb29db0e23435e3e576e54eca Mon Sep 17 00:00:00 2001 From: Johannes Weissmann Date: Thu, 18 Apr 2024 12:30:13 +0200 Subject: [PATCH 5/9] Renames "MDReport" to "ArchitectureReport" --- .../data/reports/architecture_report.py | 9 +++++++ varats/varats/data/reports/md_report.py | 11 --------- .../vara/architecture_report_experiment.py | 24 +++++++------------ 3 files changed, 17 insertions(+), 27 deletions(-) create mode 100644 varats/varats/data/reports/architecture_report.py delete mode 100644 varats/varats/data/reports/md_report.py diff --git a/varats/varats/data/reports/architecture_report.py b/varats/varats/data/reports/architecture_report.py new file mode 100644 index 000000000..72bac1d39 --- /dev/null +++ b/varats/varats/data/reports/architecture_report.py @@ -0,0 +1,9 @@ +"""Architecture report implementation.""" + +from varats.report.report import BaseReport + + +class ArchitectureReport(BaseReport, shorthand="AR", file_type="txt"): + """An metadata report for analyzing a project's architecture.""" + + # TODO Implement the ArchitectureReport class. diff --git a/varats/varats/data/reports/md_report.py b/varats/varats/data/reports/md_report.py deleted file mode 100644 index 5a047af17..000000000 --- a/varats/varats/data/reports/md_report.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Metadata report implementation.""" - -from varats.report.report import BaseReport - - -class MDReport(BaseReport, shorthand="MD", file_type="txt"): - """ - An Metadata report which serves as a base class for the Architecture report. - """ - - # TODO Implement the MDReport class. diff --git a/varats/varats/experiments/vara/architecture_report_experiment.py b/varats/varats/experiments/vara/architecture_report_experiment.py index a72647e41..abef11e18 100644 --- a/varats/varats/experiments/vara/architecture_report_experiment.py +++ b/varats/varats/experiments/vara/architecture_report_experiment.py @@ -7,7 +7,7 @@ from benchbuild.utils import actions from benchbuild.utils.cmd import touch -from varats.data.reports.md_report import MDReport +from varats.data.reports.architecture_report import ArchitectureReport from varats.experiment.experiment_util import ( VersionExperiment, ExperimentHandle, @@ -44,21 +44,16 @@ def analyze(self) -> actions.StepResult: for binary in self.project.binaries: result_file = create_new_success_result_filepath( - self.__experiment_handle, - MDReport, - self.project, binary, - config_id + self.__experiment_handle, ArchitectureReport, self.project, + binary, config_id ) opt_params = [ - "-O0", "-g0", "-fvara-GB", - "-S", "-fvara-arch", + "-O0", "-g0", "-fvara-GB", "-S", "-fvara-arch", f"-vara-report-outfile={result_file}", get_cached_bc_file_path( - self.project, binary, [ - BCFileExtensions.NO_OPT, - BCFileExtensions.ARCH - ] + self.project, binary, + [BCFileExtensions.NO_OPT, BCFileExtensions.ARCH] ) ] @@ -67,7 +62,7 @@ def analyze(self) -> actions.StepResult: exec_func_with_pe_error_handler( run_cmd, create_default_analysis_failure_handler( - self.__experiment_handle, self.project, MDReport + self.__experiment_handle, self.project, ArchitectureReport ) ) @@ -80,7 +75,7 @@ class ArchitectureReport(VersionExperiment, shorthand="ARE"): NAME = "GenerateArchitectureReport" - REPORT_SPEC = ReportSpecification(MDReport) + REPORT_SPEC = ReportSpecification(ArchitectureReport) def actions_for_project( self, project: Project @@ -107,6 +102,3 @@ def actions_for_project( analysis_actions.append(actions.Clean(project)) return analysis_actions - - - From 6e35a8f95cdab7debb6203ddbc6777653f6896bc Mon Sep 17 00:00:00 2001 From: Johannes Weissmann Date: Fri, 19 Apr 2024 14:58:38 +0200 Subject: [PATCH 6/9] Resolves the remaining comments from #878 --- .../vara/architecture_report_experiment.py | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/varats/varats/experiments/vara/architecture_report_experiment.py b/varats/varats/experiments/vara/architecture_report_experiment.py index abef11e18..6d0eea7e5 100644 --- a/varats/varats/experiments/vara/architecture_report_experiment.py +++ b/varats/varats/experiments/vara/architecture_report_experiment.py @@ -1,11 +1,12 @@ -"""Implements an empty experiment that just compiles the project.""" +"""Implements an experiment which generates an architecture report for the +project.""" import typing as tp from benchbuild import Project from benchbuild.extensions import compiler, run, time from benchbuild.utils import actions -from benchbuild.utils.cmd import touch +from benchbuild.utils.cmd import opt from varats.data.reports.architecture_report import ArchitectureReport from varats.experiment.experiment_util import ( @@ -14,9 +15,14 @@ exec_func_with_pe_error_handler, get_default_compile_error_wrapped, create_default_analysis_failure_handler, + create_default_compiler_error_handler, create_new_success_result_filepath, ) -from varats.experiment.wllvm import RunWLLVM +from varats.experiment.wllvm import ( + RunWLLVM, + BCFileExtensions, + get_bc_cache_actions, +) from varats.project.varats_project import VProject from varats.report.report import ReportSpecification from varats.utils.config import get_current_config_id @@ -35,11 +41,6 @@ def __init__(self, project: Project, experiment_handle: ExperimentHandle): self.__experiment_handle = experiment_handle def __call__(self) -> actions.StepResult: - return self.analyze() - - def analyze(self) -> actions.StepResult: - # - config_id = get_current_config_id(self.project) for binary in self.project.binaries: @@ -49,8 +50,9 @@ def analyze(self) -> actions.StepResult: ) opt_params = [ - "-O0", "-g0", "-fvara-GB", "-S", "-fvara-arch", + "--enable-new-pm=0", f"-vara-report-outfile={result_file}", + # TODO an option which enables the analysis pass on VaRA's side get_cached_bc_file_path( self.project, binary, [BCFileExtensions.NO_OPT, BCFileExtensions.ARCH] @@ -69,8 +71,7 @@ def analyze(self) -> actions.StepResult: return actions.StepResult.OK -# Please take care when changing this file, see docs experiments/just_compile -class ArchitectureReport(VersionExperiment, shorthand="ARE"): +class ArchitectureReportExperiment(VersionExperiment, shorthand="ARE"): """Generates an Architecture report file.""" NAME = "GenerateArchitectureReport" @@ -92,13 +93,26 @@ def actions_for_project( << RunWLLVM() \ << run.WithTimeout() + project.cflags += [ + "-O1", "-Xclang", "-disable-llvm-optzns", "-g0", "-fvara-arch" + ] project.compile = get_default_compile_error_wrapped( self.get_handle(), project, self.REPORT_SPEC.main_report ) - analysis_actions = [] - analysis_actions.append(actions.Compile(project)) - analysis_actions.append(EmptyAnalysis(project, self.get_handle())) + bc_file_extensions = [ + BCFileExtensions.NO_OPT, + BCFileExtentions.ARCH, + ] + extraction_error_handler = create_default_compiler_error_handler( + self.get_handle(), project, self.REPORT_SPEC.main_report + ) + analysis_actions = get_bc_cache_actions( + project, bc_file_extensions, extraction_error_handler + ) + analysis_actions.append( + ArchitectureAnalysis(project, self.get_handle()) + ) analysis_actions.append(actions.Clean(project)) return analysis_actions From 773f1bfa4a22a0283aacffaddcd17efdaf918a87 Mon Sep 17 00:00:00 2001 From: Johannes Weissmann Date: Mon, 22 Apr 2024 06:46:43 +0200 Subject: [PATCH 7/9] Fixes typing error in "architecture_report_experiment.py" --- .../varats/experiments/vara/architecture_report_experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/varats/varats/experiments/vara/architecture_report_experiment.py b/varats/varats/experiments/vara/architecture_report_experiment.py index 6d0eea7e5..d7c3d152f 100644 --- a/varats/varats/experiments/vara/architecture_report_experiment.py +++ b/varats/varats/experiments/vara/architecture_report_experiment.py @@ -102,7 +102,7 @@ def actions_for_project( bc_file_extensions = [ BCFileExtensions.NO_OPT, - BCFileExtentions.ARCH, + BCFileExtensions.ARCH, ] extraction_error_handler = create_default_compiler_error_handler( self.get_handle(), project, self.REPORT_SPEC.main_report From 53840149685115dc551bc2fc7afce902958c8b15 Mon Sep 17 00:00:00 2001 From: Johannes Weissmann Date: Mon, 22 Apr 2024 09:05:55 +0200 Subject: [PATCH 8/9] Imports a missing function --- varats/varats/experiments/vara/architecture_report_experiment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/varats/varats/experiments/vara/architecture_report_experiment.py b/varats/varats/experiments/vara/architecture_report_experiment.py index d7c3d152f..58f258246 100644 --- a/varats/varats/experiments/vara/architecture_report_experiment.py +++ b/varats/varats/experiments/vara/architecture_report_experiment.py @@ -22,6 +22,7 @@ RunWLLVM, BCFileExtensions, get_bc_cache_actions, + get_cached_bc_file_path, ) from varats.project.varats_project import VProject from varats.report.report import ReportSpecification From 56db4e7d291feea16202707c89af75c40f5f6a83 Mon Sep 17 00:00:00 2001 From: Johannes Weissmann Date: Sat, 27 Apr 2024 20:30:11 +0200 Subject: [PATCH 9/9] Updates the "projects_conf.value" array in "bb_config.py" to pass the test "test_bb_config.py" --- varats/varats/tools/bb_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/varats/varats/tools/bb_config.py b/varats/varats/tools/bb_config.py index 4999b9f48..0f0a4bcf3 100644 --- a/varats/varats/tools/bb_config.py +++ b/varats/varats/tools/bb_config.py @@ -109,6 +109,7 @@ def update_experiments(bb_cfg: s.Configuration) -> None: 'varats.experiments.szz.pydriller_szz_experiment', 'varats.experiments.szz.szz_unleashed_experiment', 'varats.experiments.vara.agg_region_interaction_perf_runner', + 'varats.experiments.vara.architecture_report_experiment', 'varats.experiments.vara.blame_report_experiment', 'varats.experiments.vara.blame_verifier_experiment', 'varats.experiments.vara.commit_report_experiment',