From 35d0d0a7242e2c5f621f3763e6339f0831e1d334 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Fri, 20 Oct 2023 14:32:34 +0200 Subject: [PATCH 01/10] Fixes dune project setup and compile options (#860) --- varats/varats/projects/cpp_projects/dune.py | 42 ++++++++++++++------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/varats/varats/projects/cpp_projects/dune.py b/varats/varats/projects/cpp_projects/dune.py index 8230febd4..4d3f58a91 100644 --- a/varats/varats/projects/cpp_projects/dune.py +++ b/varats/varats/projects/cpp_projects/dune.py @@ -2,7 +2,7 @@ import typing as tp import benchbuild as bb -from benchbuild.command import WorkloadSet, Command, SourceRoot +from benchbuild.command import WorkloadSet, SourceRoot from benchbuild.utils import cmd from benchbuild.utils.revision_ranges import RevisionRange from plumbum import local @@ -16,6 +16,8 @@ BinaryType, ProjectBinaryWrapper, ) +from varats.project.sources import FeatureSource +from varats.project.varats_command import VCommand from varats.project.varats_project import VProject from varats.utils.git_util import ShortCommitHash, RevisionBinaryMap @@ -32,7 +34,13 @@ class DunePerfRegression(VProject): If you use Dune with an experiment that uses the vara compiler, add `-mllvm --vara-disable-phasar` to the projects `cflags` to disable phasar passes. - This will still allow to analyse compile-time variability. + This will still allow to analyse compile-time variability. + + Might need deps: + * klu + * spqr + * umfpack + * eigen3 """ NAME = 'DunePerfRegression' @@ -42,25 +50,26 @@ class DunePerfRegression(VProject): SOURCE = [ PaperConfigSpecificGit( project_name='DunePerfRegression', - remote='git@github.com:se-sic/dune-VaRA.git', + remote='https://github.com/se-sic/dune-VaRA.git', local='dune-VaRA', refspec='origin/HEAD', limit=None, shallow=False - ) + ), + FeatureSource() ] CONTAINER = get_base_image(ImageBase.DEBIAN_10) WORKLOADS = { WorkloadSet(WorkloadCategory.EXAMPLE): [ - Command( + VCommand( SourceRoot( "dune-VaRA/dune-performance-regressions/build-cmake/src" ) / RSBinary('dune_performance_regressions'), label='dune_helloworld' ), - Command( + VCommand( SourceRoot( "dune-VaRA/dune-performance-regressions/build-cmake/src" ) / RSBinary('poisson_test'), @@ -71,42 +80,42 @@ class DunePerfRegression(VProject): 'poisson-yasp-Q2-3d.vtu' ] ), - Command( + VCommand( SourceRoot( "dune-VaRA/dune-performance-regressions/build-cmake/src" ) / RSBinary('poisson_ug_pk_2d'), label='poisson_ug_pk_2d', creates=['poisson-UG-Pk-2d.vtu'] ), - Command( + VCommand( SourceRoot( "dune-VaRA/dune-performance-regressions/build-cmake/src" ) / RSBinary('poisson_yasp_q1_2d'), label='poisson_yasp_q1_2d', creates=['poisson-yasp-q1-2d.vtu'] ), - Command( + VCommand( SourceRoot( "dune-VaRA/dune-performance-regressions/build-cmake/src" ) / RSBinary('poisson_yasp_q1_3d'), label='poisson_yasp_q1_3d', creates=['poisson-yasp-q1-3d.vtu'] ), - Command( + VCommand( SourceRoot( "dune-VaRA/dune-performance-regressions/build-cmake/src" ) / RSBinary('poisson_yasp_q2_2d'), label='poisson_yasp_q2_2d', creates=['poisson-yasp-q2-2d.vtu'] ), - Command( + VCommand( SourceRoot( "dune-VaRA/dune-performance-regressions/build-cmake/src" ) / RSBinary('poisson_yasp_q2_3d'), label='poisson_yasp_q2_3d', creates=['poisson-yasp-q2-3d.vtu'] ), - Command( + VCommand( SourceRoot( "dune-VaRA/dune-performance-regressions/build-cmake/src" ) / RSBinary('poisson_alugrid'), @@ -184,7 +193,14 @@ def compile(self) -> None: cxx_compiler = bb.compiler.cxx(self) with local.cwd(version_source): - with local.env(CC=c_compiler, CXX=cxx_compiler): + with local.env( + CC=c_compiler, + CXX=cxx_compiler, + CMAKE_FLAGS=" ".join([ + "-DDUNE_ENABLE_PYTHONBINDINGS=OFF", + "-DCMAKE_DISABLE_FIND_PACKAGE_MPI=TRUE" + ]) + ): dunecontrol = cmd['./dune-common/bin/dunecontrol'] bb.watch(dunecontrol From 4b38dfc3fe3803eabd632454d6bd5ee21bbdd60c Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Mon, 23 Oct 2023 14:08:46 +0200 Subject: [PATCH 02/10] Prepares support for Debian 12 containers (#854) --- tests/test_containers.py | 10 ++++--- varats/varats/containers/containers.py | 36 +++++++++++++++++++------- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/tests/test_containers.py b/tests/test_containers.py index 5125071b7..fa48a6e9b 100644 --- a/tests/test_containers.py +++ b/tests/test_containers.py @@ -32,6 +32,10 @@ class TestImageBase(unittest.TestCase): def test_distro(self) -> None: self.assertEqual(Distro.DEBIAN, ImageBase.DEBIAN_10.distro) + def test_distro_version_number(self) -> None: + self.assertEqual(10, ImageBase.DEBIAN_10.version) + self.assertEqual(12, ImageBase.DEBIAN_12.version) + class TestContainerSupport(unittest.TestCase): """Test container support related functionality.""" @@ -96,7 +100,7 @@ def test_create_stage_10_from_pip(self) -> None: self.check_layer_type(layers[0], FromLayer) varats_core_install_layer = self.check_layer_type(layers[2], RunLayer) - self.assertEqual("pip3", varats_core_install_layer.command) + self.assertEqual("pip", varats_core_install_layer.command) self.assertTupleEqual( ("install", "--ignore-installed", "varats-core", "varats"), varats_core_install_layer.args @@ -123,7 +127,7 @@ def test_create_stage_10_from_source(self) -> None: self.check_layer_type(layers[0], FromLayer) varats_core_install_layer = self.check_layer_type(layers[4], RunLayer) - self.assertEqual("pip3", varats_core_install_layer.command) + self.assertEqual("pip", varats_core_install_layer.command) self.assertTupleEqual(("install", "/varats/varats-core"), varats_core_install_layer.args) mounting_parameters = "type=bind,src=varats_src,target=/varats" @@ -133,7 +137,7 @@ def test_create_stage_10_from_source(self) -> None: varats_core_install_layer.kwargs) varats_install_layer = self.check_layer_type(layers[5], RunLayer) - self.assertEqual("pip3", varats_install_layer.command) + self.assertEqual("pip", varats_install_layer.command) self.assertTupleEqual(("install", "/varats/varats"), varats_install_layer.args) mounting_parameters = "type=bind,src=varats_src,target=/varats" diff --git a/varats/varats/containers/containers.py b/varats/varats/containers/containers.py index ddb8abe93..6e3e319c0 100644 --- a/varats/varats/containers/containers.py +++ b/varats/varats/containers/containers.py @@ -37,16 +37,23 @@ class ImageBase(Enum): """Container image bases that can be used by projects.""" - DEBIAN_10 = Distro.DEBIAN + DEBIAN_10 = (Distro.DEBIAN, 10) + DEBIAN_12 = (Distro.DEBIAN, 12) - def __init__(self, distro: Distro): + def __init__(self, distro: Distro, version_number: int): self.__distro = distro + self.__version_number = version_number @property def distro(self) -> Distro: """Distro of the base image.""" return self.__distro + @property + def version(self) -> int: + """Version number of the distro.""" + return self.__version_number + class ImageStage(Enum): """The stages that make up a base image.""" @@ -156,7 +163,7 @@ def _create_stage_00_base_layers(stage_builder: StageBuilder) -> None: def _create_stage_10_varats_layers(stage_builder: StageBuilder) -> None: - stage_builder.layers.run('pip3', 'install', '--upgrade', 'pip') + stage_builder.layers.run('pip', 'install', '--upgrade', 'pip') _add_varats_layers(stage_builder) if bb_cfg()['container']['from_source']: add_benchbuild_layers(stage_builder.layers) @@ -215,13 +222,22 @@ def wrapped(stage_builder: StageBuilder) -> None: .run('make', '-j', str(get_number_of_jobs(bb_cfg()))) .run('make', 'install') .workingdir('/') - # install llvm 13 + # install llvm 14 .run('wget', 'https://apt.llvm.org/llvm.sh') .run('chmod', '+x', './llvm.sh') .run('./llvm.sh', '14', 'all') .run('ln', '-s', '/usr/bin/clang-14', '/usr/bin/clang') .run('ln', '-s', '/usr/bin/clang++-14', '/usr/bin/clang++') - .run('ln', '-s', '/usr/bin/lld-14', '/usr/bin/lld')) + .run('ln', '-s', '/usr/bin/lld-14', '/usr/bin/lld')), + ImageBase.DEBIAN_12: + _create_layers_helper(lambda ctx: ctx.layers + .from_("docker.io/library/debian:12") + .run('apt', 'update') + .run('apt', 'install', '-y', 'wget', 'gnupg', 'lsb-release', + 'software-properties-common', 'musl-dev', 'git', 'gcc', + 'libgit2-dev', 'libffi-dev', 'libyaml-dev', 'graphviz-dev', + 'python3', 'python3-pip', 'python3-virtualenv', 'clang', + 'lld', 'time')) } _STAGE_LAYERS: tp.Dict[ImageStage, @@ -313,7 +329,9 @@ def _set_varats_source_mount(image_context: StageBuilder, mnt_src: str) -> None: def _setup_venv(image_context: StageBuilder) -> None: venv_path = "/venv" - image_context.layers.run("pip3", "install", "virtualenv") + if image_context.base == ImageBase.DEBIAN_10: + image_context.layers.run("pip3", "install", "virtualenv") + image_context.layers.run("virtualenv", venv_path) image_context.layers.env(VIRTUAL_ENV=venv_path) image_context.layers.env(PATH=f"{venv_path}/bin:$PATH") @@ -331,9 +349,9 @@ def from_source( tgt_dir = image_context.varats_source_mount_target image.run('mkdir', f'{tgt_dir}', runtime=crun) - image.run('pip3', 'install', 'setuptools', runtime=crun) + image.run('pip', 'install', 'setuptools', runtime=crun) - pip_args = ['pip3', 'install'] + pip_args = ['pip', 'install'] if editable_install: pip_args.append("-e") _set_varats_source_mount(image_context, str(src_dir)) @@ -348,7 +366,7 @@ def from_source( def from_pip(image: ContainerImage) -> None: LOG.debug("installing varats from pip release.") image.run( - 'pip3', + 'pip', 'install', '--ignore-installed', 'varats-core', From 92903b80b941c4cd5bbe09377deee244fc09bc54 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Mon, 23 Oct 2023 14:18:17 +0200 Subject: [PATCH 03/10] Fixes BB config init when running buildsetup (#851) --- varats/varats/tools/driver_build_setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/varats/varats/tools/driver_build_setup.py b/varats/varats/tools/driver_build_setup.py index 6023d9104..78e2dbbe1 100644 --- a/varats/varats/tools/driver_build_setup.py +++ b/varats/varats/tools/driver_build_setup.py @@ -270,6 +270,7 @@ def _build_in_container( install_mount = 'tools/' click.echo("Preparing container image.") + bb_cfg() # Ensure that BB config is loaded image_name = create_dev_image(image_base, build_type) source_mount = str(StageBuilder.varats_root / source_mount) From 19212e19aa061e60bbaa338120eb0ff4e93a3c91 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Mon, 23 Oct 2023 14:34:48 +0200 Subject: [PATCH 04/10] Fixes threshold handling for feature experiments (#853) --- varats/varats/experiments/vara/feature_experiment.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/varats/varats/experiments/vara/feature_experiment.py b/varats/varats/experiments/vara/feature_experiment.py index fd4ce236e..a4cc8722d 100644 --- a/varats/varats/experiments/vara/feature_experiment.py +++ b/varats/varats/experiments/vara/feature_experiment.py @@ -17,6 +17,7 @@ Compile, Clean, ) +from benchbuild.utils.requirements import Requirement, SlurmMem from plumbum import local from varats.experiment.experiment_util import ( @@ -72,6 +73,8 @@ class FeatureExperiment(VersionExperiment, shorthand=""): REPORT_SPEC = ReportSpecification() + REQUIREMENTS: tp.List[Requirement] = [SlurmMem("250G")] + @abstractmethod def actions_for_project(self, project: VProject) -> tp.MutableSequence[Step]: @@ -180,18 +183,23 @@ def get_vara_tracing_cflags( Returns: list of tracing specific cflags """ c_flags = [] + if instr_type != FeatureInstrType.NONE: c_flags += ["-fsanitize=vara", f"-fvara-instr={instr_type.value}"] + c_flags += [ "-flto", "-fuse-ld=lld", "-flegacy-pass-manager", "-fno-omit-frame-pointer" ] - if instruction_threshold is not None: + + if instruction_threshold is None: # For test projects, do not exclude small regions if project is not None and project.domain == ProjectDomains.TEST: instruction_threshold = 1 + if instruction_threshold is not None: c_flags += [f"-fvara-instruction-threshold={instruction_threshold}"] + if save_temps: c_flags += ["-Wl,-plugin-opt=save-temps"] @@ -230,7 +238,7 @@ def __call__(self) -> StepResult: def __str__(self, indent: int = 0) -> str: return textwrap.indent( - f"* {self.project.name}: Run instrumentation verifier", indent * " " + f"* {self.project.name}: Run instrumented code", indent * " " ) def run_traced_code(self) -> StepResult: From 0355a334e86e2e717152720a45895a47652856eb Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Mon, 23 Oct 2023 14:45:28 +0200 Subject: [PATCH 05/10] Adds linux perf report (#857) --- tests/report/test_linux_perf_report.py | 102 +++++++++++++++ .../varats/report/linux_perf_report.py | 117 ++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 tests/report/test_linux_perf_report.py create mode 100644 varats-core/varats/report/linux_perf_report.py diff --git a/tests/report/test_linux_perf_report.py b/tests/report/test_linux_perf_report.py new file mode 100644 index 000000000..19f3d0781 --- /dev/null +++ b/tests/report/test_linux_perf_report.py @@ -0,0 +1,102 @@ +"""Test LinuxPerfReport.""" + +import unittest +from pathlib import Path +from unittest import mock + +from varats.report.linux_perf_report import LinuxPerfReport + +PERF_REPORT_1 = """# started on Sun Jul 23 22:51:54 2023 + + + Performance counter stats for 'echo foo:bar': + + 0.30 msec task-clock:u # 0.406 CPUs utilized + 0 context-switches:u # 0.000 /sec + 0 cpu-migrations:u # 0.000 /sec + 64 page-faults:u # 212.723 K/sec + 360,721 cycles:u # 1.199 GHz + 26,199 stalled-cycles-frontend:u # 7.26% frontend cycles idle + 111,008 stalled-cycles-backend:u # 30.77% backend cycles idle + 200,655 instructions:u # 0.56 insn per cycle + # 0.55 stalled cycles per insn + 48,631 branches:u # 161.639 M/sec + 3,012 branch-misses:u # 6.19% of all branches + L1-dcache-loads:u (0.00%) + L1-dcache-load-misses:u (0.00%) + LLC-loads:u + LLC-load-misses:u + + 0.000741511 seconds time elapsed + + 0.000000000 seconds user + 0.000822000 seconds sys + + + +""" + +PERF_REPORT_2 = """# started on Sun Jul 23 22:44:31 2023 + + + Performance counter stats for 'foobar': + + 1.23 msec task-clock:u # 0.000 CPUs utilized + 0 context-switches:u # 0.000 /sec + 0 cpu-migrations:u # 0.000 /sec + 132 page-faults:u # 107.572 K/sec + 850,975 cycles:u # 0.693 GHz (12.81%) + 140,154 stalled-cycles-frontend:u # 16.47% frontend cycles idle + 1,012,322 stalled-cycles-backend:u # 118.96% backend cycles idle + 1,785,912 instructions:u # 2.10 insn per cycle + # 0.57 stalled cycles per insn + 325,708 branches:u # 265.433 M/sec + 11,160 branch-misses:u # 3.43% of all branches + 840,918 L1-dcache-loads:u # 685.298 M/sec (87.19%) + L1-dcache-load-misses:u (0.00%) + LLC-loads:u + LLC-load-misses:u + + 5.945920439 seconds time elapsed + + 0.000376000 seconds user + 0.001390000 seconds sys + + + +""" + + +class TestLinuxPerfReport(unittest.TestCase): + """Tests if the Linux perf report can be loaded correctly.""" + + report_1: LinuxPerfReport + report_2: LinuxPerfReport + + @classmethod + def setUpClass(cls) -> None: + """Load Linux perf report.""" + with mock.patch( + "builtins.open", new=mock.mock_open(read_data=PERF_REPORT_1) + ): + cls.report_1 = LinuxPerfReport(Path("fake_file_path")) + + with mock.patch( + "builtins.open", new=mock.mock_open(read_data=PERF_REPORT_2) + ): + cls.report_2 = LinuxPerfReport(Path("fake_file_path")) + + def test_task_clock_parsing(self) -> None: + """Checks if we correctly parsed the value for task clock.""" + self.assertEqual(self.report_1.elapsed_time, 0.000741511) + self.assertEqual(self.report_2.elapsed_time, 5.945920439) + + def test_context_switches_parsing(self) -> None: + """Checks if we correctly parsed the value for context switches.""" + self.assertEqual(self.report_1.ctx_switches, 0) + self.assertEqual(self.report_2.ctx_switches, 0) + + def test_branch_misses_parsing(self) -> None: + """Checks if we correctly parsed the value for branch misses.""" + self.assertEqual(self.report_1.branch_misses, 3012) + self.assertEqual(self.report_2.branch_misses, 11160) diff --git a/varats-core/varats/report/linux_perf_report.py b/varats-core/varats/report/linux_perf_report.py new file mode 100644 index 000000000..ef0eacdea --- /dev/null +++ b/varats-core/varats/report/linux_perf_report.py @@ -0,0 +1,117 @@ +""" +Simple report module to create and handle the standard timing output of perf +stat. + +Examples to produce a ``LinuxPerfReport``: + + Commandline usage: + .. code-block:: bash + + export REPORT_FILE="Path/To/MyFile" + perf stat -x ";" -o $REPORT_FILE -- sleep 2 + + Experiment code: + .. code-block:: python + + from benchbuild.utils.cmd import time, sleep + report_file = "Path/To/MyFile" + command = sleep["2"] + perf("stat", "-x", "';'", "-o", f"{report_file}", "--", command) +""" +import math +import typing as tp +from pathlib import Path + +import numpy as np + +from varats.report.report import BaseReport, ReportAggregate + + +class LinuxPerfReport(BaseReport, shorthand="LPR", file_type="txt"): + """Report class to access perf stat output.""" + + def __init__(self, path: Path) -> None: + super().__init__(path) + self.__elapsed_time = math.nan + self.__ctx_switches: int = -1 + self.__branch_misses: int = -1 + + with open(self.path, 'r', newline="") as stream: + for line in stream: + line = line.strip("\n ") + # print(f"{line=}") + + if line == "" or line.startswith("#"): + continue + + if "time elapsed" in line: + self.__elapsed_time = self.__parse_elapsed_time(line) + + if "context-switches:u" in line: + self.__ctx_switches = self.__parse_ctx_switches(line) + + if "branch-misses:u" in line: + self.__branch_misses = self.__parse_branch_misses(line) + + if self.__branch_misses == math.nan: + raise AssertionError() + + @staticmethod + def __parse_elapsed_time(line: str) -> float: + return float(line.split(" ")[0].replace(",", "")) + + @staticmethod + def __parse_ctx_switches(line: str) -> int: + return int(line.split(" ")[0].replace(",", "")) + + @staticmethod + def __parse_branch_misses(line: str) -> tp.Optional[int]: + if line.startswith(""): + return None + return int(line.split(" ")[0].replace(",", "")) + + @property + def elapsed_time(self) -> float: + return self.__elapsed_time + + @property + def ctx_switches(self) -> int: + return self.__ctx_switches + + @property + def branch_misses(self) -> int: + return self.__branch_misses + + def __repr__(self) -> str: + return str(self) + + def __str__(self) -> str: + return f"""LPR ({self.path}) + ├─ ElapsedTime: {self.elapsed_time} + ├─ CtxSwitches: {self.ctx_switches} + └─ BranchMisses: {self.branch_misses} +""" + + +class LinuxPerfReportAggregate( + ReportAggregate[LinuxPerfReport], + shorthand=LinuxPerfReport.SHORTHAND + ReportAggregate.SHORTHAND, + file_type=ReportAggregate.FILE_TYPE +): + """Meta report for parsing multiple Linux perf reports stored inside a zip + file.""" + + def __init__(self, path: Path) -> None: + super().__init__(path, LinuxPerfReport) + + @property + def elapsed_time(self) -> tp.List[float]: + return [report.elapsed_time for report in self.reports()] + + @property + def ctx_switches(self) -> tp.List[int]: + return [report.ctx_switches for report in self.reports()] + + @property + def branch_misses(self) -> tp.List[int]: + return [report.branch_misses for report in self.reports()] From 8f13f3a3dfd1f3c48875a31d9b25a380a6b40449 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Mon, 23 Oct 2023 14:58:54 +0200 Subject: [PATCH 06/10] Extends gnu time report (#856) --- tests/report/test_gnu_time_report.py | 36 ++++++++- varats-core/varats/report/gnu_time_report.py | 80 +++++++++++++++++++- 2 files changed, 111 insertions(+), 5 deletions(-) diff --git a/tests/report/test_gnu_time_report.py b/tests/report/test_gnu_time_report.py index 80a74b550..d836c1929 100644 --- a/tests/report/test_gnu_time_report.py +++ b/tests/report/test_gnu_time_report.py @@ -18,13 +18,13 @@ Average total size (kbytes): 0 Maximum resident set size (kbytes): 1804 Average resident set size (kbytes): 0 - Major (requiring I/O) page faults: 0 + Major (requiring I/O) page faults: 2 Minor (reclaiming a frame) page faults: 142 Voluntary context switches: 1 Involuntary context switches: 1 Swaps: 0 - File system inputs: 0 - File system outputs: 0 + File system inputs: 1 + File system outputs: 2 Socket messages sent: 0 Socket messages received: 0 Signals delivered: 0 @@ -63,6 +63,12 @@ def test_max_resident_size(self): with self.assertRaises(WrongTimeReportFormat): TimeReport._parse_max_resident_size(" Something other timed:") + def test_major_page_faults(self): + """Test if we correctly parse the amount of major page faults from the + input line.""" + with self.assertRaises(WrongTimeReportFormat): + TimeReport._parse_major_page_faults(" Something other timed:") + def test_max_resident_size_byte_type(self): """Test if we correctly parse the max resident size from the input line.""" @@ -97,6 +103,30 @@ def test_system_time(self): """Test if we can extract the system time from the parsed file.""" self.assertEqual(self.report.system_time, timedelta(seconds=3)) + def test_wall_clock_time(self): + """Test if we can extract the wall clock time from the parsed file.""" + self.assertEqual(self.report.wall_clock_time, timedelta(seconds=42)) + + def test_max_resident_size(self) -> None: + """Test if we can extract the max resident size from the parsed file.""" + self.assertEqual(self.report.max_res_size, 1804) + + def test_major_page_faults(self) -> None: + """Test if we can extract the number of major page faults from the + parsed file.""" + self.assertEqual(self.report.major_page_faults, 2) + + def test_minor_page_faults(self) -> None: + """Test if we can extract the number of minor page faults from the + parsed file.""" + self.assertEqual(self.report.minor_page_faults, 142) + + def test_filesystem_io(self) -> None: + """Test if we can extract the number of filesystem inputs/outputs from + the parsed file.""" + self.assertEqual(self.report.filesystem_io[0], 1) + self.assertEqual(self.report.filesystem_io[1], 2) + def test_repr_str(self): """Test string representation of TimeReports.""" expected_result = """Command: echo diff --git a/varats-core/varats/report/gnu_time_report.py b/varats-core/varats/report/gnu_time_report.py index 124929602..ddaeecac3 100644 --- a/varats-core/varats/report/gnu_time_report.py +++ b/varats-core/varats/report/gnu_time_report.py @@ -37,7 +37,7 @@ class TimeReport(BaseReport, shorthand="TR", file_type="txt"): def __init__(self, path: Path) -> None: super().__init__(path) - + self.__filesystem_io = (-1, -1) with open(self.path, 'r') as stream: for line in stream: line = line.strip() @@ -64,6 +64,16 @@ def __init__(self, path: Path) -> None: TimeReport._parse_wall_clock_time(line) continue + if line.startswith("Major (requiring I/O) page faults"): + self.__major_page_faults: int = \ + TimeReport._parse_major_page_faults(line) + continue + + if line.startswith("Minor (reclaiming a frame) page faults"): + self.__minor_page_faults: int = \ + TimeReport._parse_minor_page_faults(line) + continue + if line.startswith("Voluntary context switches"): self.__voluntary_ctx_switches: int = \ TimeReport._parse_voluntary_ctx_switches(line) @@ -74,7 +84,19 @@ def __init__(self, path: Path) -> None: TimeReport._parse_involuntary_ctx_switches(line) continue - # print("Not matched: ", line) + if line.startswith("File system inputs"): + self.__filesystem_io = ( + TimeReport._parse_filesystem_io(line), + self.__filesystem_io[1] + ) + continue + + if line.startswith("File system outputs"): + self.__filesystem_io = ( + self.__filesystem_io[0], + TimeReport._parse_filesystem_io(line) + ) + continue @property def command_name(self) -> str: @@ -101,6 +123,25 @@ def max_res_size(self) -> int: """Maximum resident size.""" return self.__max_resident_size + @property + def major_page_faults(self) -> int: + """Major page faults (require I/O).""" + return self.__major_page_faults + + @property + def minor_page_faults(self) -> int: + """Minor page faults (reclaim a frame).""" + return self.__minor_page_faults + + @property + def filesystem_io(self) -> tp.Tuple[int, int]: + """ + Filesystem inputs/outputs. + + Returns: a tuple of (#inputs, #outputs) + """ + return self.__filesystem_io + @property def voluntary_ctx_switches(self) -> int: """Number of voluntary context switches.""" @@ -217,6 +258,20 @@ def _parse_max_resident_size(line: str) -> int: "Could not parse max resident set size: ", line ) + @staticmethod + def _parse_major_page_faults(line: str) -> int: + if line.startswith("Major (requiring I/O) page faults"): + return int(line.split(":")[1]) + + raise WrongTimeReportFormat("Could not parse major page faults: ", line) + + @staticmethod + def _parse_minor_page_faults(line: str) -> int: + if line.startswith("Minor (reclaiming a frame) page faults"): + return int(line.split(":")[1]) + + raise WrongTimeReportFormat("Could not parse minor page faults: ", line) + @staticmethod def _parse_voluntary_ctx_switches(line: str) -> int: if line.startswith("Voluntary context switches"): @@ -235,6 +290,15 @@ def _parse_involuntary_ctx_switches(line: str) -> int: "Could not parse involuntary context switches: ", line ) + @staticmethod + def _parse_filesystem_io(line: str) -> int: + if line.startswith("File system "): + return int(line.split(":")[1]) + + raise WrongTimeReportFormat( + "Could not parse filesystem inputs/outputs: ", line + ) + class TimeReportAggregate( ReportAggregate[TimeReport], @@ -268,6 +332,18 @@ def measurements_ctx_switches(self) -> tp.List[int]: def max_resident_sizes(self) -> tp.List[int]: return [report.max_res_size for report in self.reports()] + @property + def major_page_faults(self) -> tp.List[int]: + return [report.major_page_faults for report in self.reports()] + + @property + def minor_page_faults(self) -> tp.List[int]: + return [report.minor_page_faults for report in self.reports()] + + @property + def filesystem_io(self) -> tp.List[tp.Tuple[int, int]]: + return [report.filesystem_io for report in self.reports()] + @property def summary(self) -> str: import numpy as np # pylint: disable=import-outside-toplevel From 9f17737c817bca48e84398590526bcebd74b4345 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Mon, 23 Oct 2023 15:35:16 +0200 Subject: [PATCH 07/10] Fixes bzip2 project setup and workloads (#859) --- varats/varats/projects/c_projects/bzip2.py | 63 +++++++++++++++++++--- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/varats/varats/projects/c_projects/bzip2.py b/varats/varats/projects/c_projects/bzip2.py index fa9de73d7..150e364a8 100644 --- a/varats/varats/projects/c_projects/bzip2.py +++ b/varats/varats/projects/c_projects/bzip2.py @@ -3,7 +3,7 @@ from pathlib import Path import benchbuild as bb -from benchbuild.command import Command, SourceRoot, WorkloadSet +from benchbuild.command import SourceRoot, WorkloadSet from benchbuild.source import HTTPMultiple from benchbuild.utils.cmd import cmake, make from benchbuild.utils.revision_ranges import RevisionRange, GoodBadSubgraph @@ -20,6 +20,8 @@ BinaryType, verify_binaries, ) +from varats.project.sources import FeatureSource +from varats.project.varats_command import VCommand from varats.project.varats_project import VProject from varats.utils.git_util import ( ShortCommitHash, @@ -56,6 +58,20 @@ class Bzip2(VProject): "countries-land-1m.geo.json", "countries-land-10m.geo.json", "countries-land-100m.geo.json" ] + ), + FeatureSource(), + HTTPMultiple( + local="geo-maps-compr", + remote={ + "1.0": + "https://github.com/se-sic/compression-data/" + "raw/master/bzip2/geo-maps/" + }, + files=[ + "countries-land-100m.geo.json.bz2", + "countries-land-10m.geo.json.bz2", + "countries-land-1m.geo.json.bz2" + ] ) ] _AUTOTOOLS_VERSIONS = GoodBadSubgraph([ @@ -80,11 +96,8 @@ class Bzip2(VProject): WORKLOADS = { WorkloadSet(WorkloadCategory.MEDIUM): [ - Command( + VCommand( SourceRoot("bzip2") / RSBinary("bzip2"), - "--compress", - "--best", - "-vvv", "--keep", # bzip2 compresses very fast even on the best setting, so we # need the three input files to get approximately 30 seconds @@ -92,11 +105,30 @@ class Bzip2(VProject): "geo-maps/countries-land-1m.geo.json", "geo-maps/countries-land-10m.geo.json", "geo-maps/countries-land-100m.geo.json", + label="med_geo", creates=[ "geo-maps/countries-land-1m.geo.json.bz2", "geo-maps/countries-land-10m.geo.json.bz2", "geo-maps/countries-land-100m.geo.json.bz2" - ] + ], + requires_all_args={"--compress"} + ), + VCommand( + SourceRoot("bzip2") / RSBinary("bzip2"), + "--keep", + # bzip2 compresses very fast even on the best setting, so we + # need the three input files to get approximately 30 seconds + # total execution time + "geo-maps-compr/countries-land-1m.geo.json.bz2", + "geo-maps-compr/countries-land-10m.geo.json.bz2", + "geo-maps-compr/countries-land-100m.geo.json.bz2", + label="med_geo", + creates=[ + "geo-maps-compr/countries-land-1m.geo.json", + "geo-maps-compr/countries-land-10m.geo.json", + "geo-maps-compr/countries-land-100m.geo.json" + ], + requires_all_args={"--decompress"} ) ], } @@ -161,3 +193,22 @@ def compile(self) -> None: ) with local.cwd(bzip2_source): verify_binaries(self) + + def recompile(self) -> None: + """Recompile the project.""" + bzip2_source = Path(self.source_of_primary) + bzip2_version = ShortCommitHash(self.version_of_primary) + + if bzip2_version in typed_revision_range( + Bzip2._MAKE_VERSIONS, bzip2_source, ShortCommitHash + ) or bzip2_version in typed_revision_range( + Bzip2._AUTOTOOLS_VERSIONS, bzip2_source, ShortCommitHash + ): + with local.cwd(bzip2_source / "build"): + bb.watch(make)("-j", get_number_of_jobs(bb_cfg())) + else: + with local.cwd(bzip2_source / "build"): + bb.watch(cmake)( + "--build", ".", "--config", "Release", "-j", + get_number_of_jobs(bb_cfg()) + ) From 13f089b3cbcfe884a8f54bf36d7c30b773adc306 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Mon, 23 Oct 2023 17:44:33 +0200 Subject: [PATCH 08/10] Adds option to pass on debug flags from vara-run to benchbuild (#850) --- varats/varats/tools/driver_run.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/varats/varats/tools/driver_run.py b/varats/varats/tools/driver_run.py index 854b31c9f..7c6ac4e0e 100644 --- a/varats/varats/tools/driver_run.py +++ b/varats/varats/tools/driver_run.py @@ -90,6 +90,9 @@ def __validate_project_parameters( @click.option( "--container", is_flag=True, help="Run experiments in a container." ) +@click.option( + "--debug", is_flag=True, help="Run container in an interactive debug mode." +) @click.option( "-E", "--experiment", @@ -104,6 +107,7 @@ def main( slurm: bool, submit: bool, container: bool, + debug: bool, experiment: tp.List[tp.Type['VersionExperiment']], projects: tp.List[str], pretend: bool, @@ -145,6 +149,9 @@ def main( bb_extra_args.append("--import") else: bb_command_args.append("container") + if debug: + bb_extra_args.append("--debug") + bb_extra_args.append("--interactive") if not slurm: bb_command_args.append("run") From 11df5460d65e7a07be74fbee2ec061817fbea46e Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Mon, 23 Oct 2023 17:54:27 +0200 Subject: [PATCH 09/10] Fixes MultiStep report to always have a base field (#858) --- varats-core/varats/report/multi_patch_report.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/varats-core/varats/report/multi_patch_report.py b/varats-core/varats/report/multi_patch_report.py index e0f7a1d5b..74f861fce 100644 --- a/varats-core/varats/report/multi_patch_report.py +++ b/varats-core/varats/report/multi_patch_report.py @@ -18,6 +18,7 @@ class MultiPatchReport( def __init__(self, path: Path, report_type: tp.Type[ReportTy]) -> None: super().__init__(path) self.__patched_reports: tp.Dict[str, ReportTy] = {} + self.__base = None with tempfile.TemporaryDirectory() as tmp_result_dir: shutil.unpack_archive(path, extract_dir=tmp_result_dir) @@ -33,7 +34,7 @@ def __init__(self, path: Path, report_type: tp.Type[ReportTy]) -> None: if not self.__base or not self.__patched_reports: raise AssertionError( - "Reports where missing in the file {report_path=}" + f"Reports where missing in the file {path=}" ) def get_baseline_report(self) -> ReportTy: From 6ee52bad293a30e4cdae142f4fc315410c9c216f Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Mon, 23 Oct 2023 20:21:53 +0200 Subject: [PATCH 10/10] Extends and fixes minor plotting/table issues (#852) --- varats/varats/plots/scatter_plot_utils.py | 3 ++- varats/varats/tables/time_workloads.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/varats/varats/plots/scatter_plot_utils.py b/varats/varats/plots/scatter_plot_utils.py index 3510d3da4..005dba2ed 100644 --- a/varats/varats/plots/scatter_plot_utils.py +++ b/varats/varats/plots/scatter_plot_utils.py @@ -13,6 +13,7 @@ def multivariate_grid( y: str, hue: str, global_kde: bool = True, + legend: bool = True, **kwargs: tp.Any ) -> sns.JointGrid: """ @@ -84,7 +85,7 @@ def multivariate_grid( color='grey', warn_singular=False ) - if len(grouped_data) > 1: + if len(grouped_data) > 1 and legend: plt.legend(legends) return grid diff --git a/varats/varats/tables/time_workloads.py b/varats/varats/tables/time_workloads.py index 4120c5210..234e8254a 100644 --- a/varats/varats/tables/time_workloads.py +++ b/varats/varats/tables/time_workloads.py @@ -95,7 +95,11 @@ def wall_clock_time_in_msecs( kwargs["column_format"] = "llr|rr|r|r" return dataframe_to_table( - df, table_format, wrap_table, wrap_landscape=True, **kwargs + df, + table_format, + wrap_table=wrap_table, + wrap_landscape=True, + **kwargs )