From 3dba6c10e741d638df092764f043a61fb2be0c8f Mon Sep 17 00:00:00 2001 From: Leonie von Mann Date: Fri, 29 Nov 2024 12:17:59 +0100 Subject: [PATCH 1/4] Add blame server experiment --- .../vara/blame_server_experiment.py | 99 +++++++++++++++++++ varats/varats/tools/bb_config.py | 1 + 2 files changed, 100 insertions(+) create mode 100644 varats/varats/experiments/vara/blame_server_experiment.py diff --git a/varats/varats/experiments/vara/blame_server_experiment.py b/varats/varats/experiments/vara/blame_server_experiment.py new file mode 100644 index 000000000..9d1877ecf --- /dev/null +++ b/varats/varats/experiments/vara/blame_server_experiment.py @@ -0,0 +1,99 @@ +""" +Implements blame experiment using a blame cache server. + +The experiment starts the blame server, compiles with blame annotations and +kills the server at the end. +""" +import socket +import typing as tp + +from benchbuild import Project +from benchbuild.utils import actions +from plumbum import local +from plumbum.cmd import kill + +import varats.experiments.vara.blame_experiment as BE +from varats.data.reports.blame_annotations import BlameAnnotations as BA +from varats.experiment.experiment_util import ( + VersionExperiment, + create_default_compiler_error_handler, +) +from varats.experiment.wllvm import BCFileExtensions, Extract +from varats.experiments.vara.blame_ast_experiment import ( + BlameAnnotationGeneration, +) +from varats.report.report import ReportSpecification + + +class BlameServerSteps(actions.Compile): # type: ignore + NAME = "BlameServerSteps" + DESCRIPTION = "Start server before and kill server after compilation." + + def __init__(self, project: Project, port: int): + super().__init__(project) + self.__port = port + + def __call__(self) -> actions.StepResult: + server_cmd = local["vara-blamed"][ + f"--blame-server=0.0.0.0:{self.__port}"] + server_proc = server_cmd.popen() + try: + steps = super().__call__() + finally: + kill[str(server_proc.pid)]() + return steps + + @staticmethod + def find_open_port(): + """Finds and returns an available port on the local machine.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(('', 0)) # Bind to a free port provided by the OS + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + + +class BlameServerExperiment(VersionExperiment, shorthand="BSE"): + """Generate a blame annotation report using blame server.""" + + NAME = "RunBlameServer" + + REPORT_SPEC = ReportSpecification(BA) + + def actions_for_project( + self, project: Project + ) -> tp.MutableSequence[actions.Step]: + """ + Returns the specified steps to run the project(s) specified in the call + in a fixed order. + + Args: + project: to analyze + """ + project.cflags += ["-O1", "-Xclang", "-disable-llvm-optzns", "-g"] + bc_file_extensions = [ + BCFileExtensions.NO_OPT, + BCFileExtensions.TBAA, + BCFileExtensions.BLAME #TODO: add extension for blame server ? + ] + + BE.setup_basic_blame_experiment(self, project, BA) + open_port = BlameServerSteps.find_open_port() + project.cflags += [f"-fvara-blame-server=0.0.0.0:{open_port}"] + analysis_actions = [] + analysis_actions.append(BlameServerSteps(project, open_port)) + analysis_actions.append( + Extract( + project, + bc_file_extensions, + handler=create_default_compiler_error_handler( + self.get_handle(), project, self.REPORT_SPEC.main_report + ) + ) + ) + analysis_actions.append( + BlameAnnotationGeneration( + project, self.get_handle(), bc_file_extensions + ) + ) + + return analysis_actions diff --git a/varats/varats/tools/bb_config.py b/varats/varats/tools/bb_config.py index b4d9237a7..11b12b9c5 100644 --- a/varats/varats/tools/bb_config.py +++ b/varats/varats/tools/bb_config.py @@ -114,6 +114,7 @@ def update_experiments(bb_cfg: s.Configuration) -> None: 'varats.experiments.vara.agg_region_interaction_perf_runner', 'varats.experiments.vara.blame_ast_experiment', 'varats.experiments.vara.blame_report_experiment', + 'varats.experiments.vara.blame_server_experiment', 'varats.experiments.vara.blame_verifier_experiment', 'varats.experiments.vara.commit_report_experiment', 'varats.experiments.vara.feature_perf_runner', From fa45695e5be5bfb06439cd23b45128f62ebbd867 Mon Sep 17 00:00:00 2001 From: Leonie von Mann Date: Sat, 21 Dec 2024 16:20:02 +0100 Subject: [PATCH 2/4] Adjustments to blame server experiment --- varats/varats/experiments/vara/blame_server_experiment.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/varats/varats/experiments/vara/blame_server_experiment.py b/varats/varats/experiments/vara/blame_server_experiment.py index 9d1877ecf..c6f2d6d95 100644 --- a/varats/varats/experiments/vara/blame_server_experiment.py +++ b/varats/varats/experiments/vara/blame_server_experiment.py @@ -26,6 +26,8 @@ class BlameServerSteps(actions.Compile): # type: ignore + """"Start blame server before and kill server after compilation.""" + NAME = "BlameServerSteps" DESCRIPTION = "Start server before and kill server after compilation." @@ -37,6 +39,7 @@ def __call__(self) -> actions.StepResult: server_cmd = local["vara-blamed"][ f"--blame-server=0.0.0.0:{self.__port}"] server_proc = server_cmd.popen() + steps = actions.StepResult(3) try: steps = super().__call__() finally: @@ -44,7 +47,7 @@ def __call__(self) -> actions.StepResult: return steps @staticmethod - def find_open_port(): + def find_open_port() -> int: """Finds and returns an available port on the local machine.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(('', 0)) # Bind to a free port provided by the OS From 628e2865a5ef8d2b07e807ae20db70451f45de59 Mon Sep 17 00:00:00 2001 From: Leonie von Mann Date: Sat, 21 Dec 2024 16:20:28 +0100 Subject: [PATCH 3/4] Add blame server experiment to docs --- .../experiments/blame_report_experiments.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/source/vara-ts-api/experiments/blame_report_experiments.rst b/docs/source/vara-ts-api/experiments/blame_report_experiments.rst index ec154e297..c04d4aeee 100644 --- a/docs/source/vara-ts-api/experiments/blame_report_experiments.rst +++ b/docs/source/vara-ts-api/experiments/blame_report_experiments.rst @@ -28,3 +28,13 @@ Module: BlameExperiment :members: :undoc-members: :show-inheritance: + +------ + +Module: BlameServerExperiment +----------------------------- + +.. automodule:: varats.experiments.vara.blame_server_experiment + :members: + :undoc-members: + :show-inheritance: From 0b1cd8ef03454db15375f771d45083c78c2fbdf3 Mon Sep 17 00:00:00 2001 From: Leonie von Mann Date: Sat, 21 Dec 2024 16:28:15 +0100 Subject: [PATCH 4/4] Fix return type annotation --- varats/varats/experiments/vara/blame_server_experiment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/varats/varats/experiments/vara/blame_server_experiment.py b/varats/varats/experiments/vara/blame_server_experiment.py index c6f2d6d95..2c80eaafb 100644 --- a/varats/varats/experiments/vara/blame_server_experiment.py +++ b/varats/varats/experiments/vara/blame_server_experiment.py @@ -26,7 +26,7 @@ class BlameServerSteps(actions.Compile): # type: ignore - """"Start blame server before and kill server after compilation.""" + """Start blame server before and kill server after compilation.""" NAME = "BlameServerSteps" DESCRIPTION = "Start server before and kill server after compilation." @@ -47,7 +47,7 @@ def __call__(self) -> actions.StepResult: return steps @staticmethod - def find_open_port() -> int: + def find_open_port() -> tp.Any: """Finds and returns an available port on the local machine.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(('', 0)) # Bind to a free port provided by the OS