From 47e32e2c506fa49058631f95bc0e98c86059fe98 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Tue, 28 Nov 2023 11:18:44 +0100 Subject: [PATCH 1/3] Adds config support for revision look ups (#855) --- tests/revision/__init__.py | 0 tests/revision/test_revisions.py | 34 ++++++++++++++++ varats-core/varats/revision/revisions.py | 50 ++++++++++++++++++------ 3 files changed, 73 insertions(+), 11 deletions(-) create mode 100644 tests/revision/__init__.py create mode 100644 tests/revision/test_revisions.py diff --git a/tests/revision/__init__.py b/tests/revision/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/revision/test_revisions.py b/tests/revision/test_revisions.py new file mode 100644 index 000000000..e54c02d97 --- /dev/null +++ b/tests/revision/test_revisions.py @@ -0,0 +1,34 @@ +"""Test revision helper functions.""" + +import unittest + +from tests.helper_utils import run_in_test_environment, UnitTestFixtures +from varats.paper.paper_config import load_paper_config +from varats.revision.revisions import get_processed_revisions_files +from varats.utils.settings import vara_cfg + + +class TestGetProcessedRevisionsFiles(unittest.TestCase): + """Test if the revision look up works correctly.""" + + @run_in_test_environment( + UnitTestFixtures.PAPER_CONFIGS, UnitTestFixtures.RESULT_FILES + ) + def test_config_specific_lookup(self) -> None: + """Check whether the config specific file loading works.""" + vara_cfg()['paper_config']['current_config'] = "test_config_ids" + load_paper_config() + + processed_rev_files_cid_1 = get_processed_revisions_files( + "SynthSAContextSensitivity", config_id=1 + ) + self.assertEqual(len(processed_rev_files_cid_1), 1) + self.assertEqual( + processed_rev_files_cid_1[0].report_filename.config_id, 1 + ) + + processed_rev_files_cid_2 = get_processed_revisions_files( + "SynthSAContextSensitivity", config_id=2 + ) + # there should not be a report for config id 2 + self.assertEqual(len(processed_rev_files_cid_2), 0) diff --git a/varats-core/varats/revision/revisions.py b/varats-core/varats/revision/revisions.py index 04d987d63..2f50f63ea 100644 --- a/varats-core/varats/revision/revisions.py +++ b/varats-core/varats/revision/revisions.py @@ -123,7 +123,8 @@ def __get_files_with_status( experiment_type: tp.Optional[tp.Type["exp_u.VersionExperiment"]] = None, report_type: tp.Optional[tp.Type[BaseReport]] = None, file_name_filter: tp.Callable[[str], bool] = lambda x: False, - only_newest: bool = True + only_newest: bool = True, + config_id: tp.Optional[int] = None ) -> tp.List[ReportFilepath]: """ Find all file paths to result files with given file statuses. @@ -148,7 +149,15 @@ def __get_files_with_status( result_files = __get_result_files_dict( project_name, experiment_type, report_type ) + for value in result_files.values(): + if config_id is not None: + value = [ + x for x in value if x.report_filename.config_id == config_id + ] + if not value: + continue + sorted_res_files = sorted( value, key=lambda x: x.stat().st_mtime, reverse=True ) @@ -168,7 +177,8 @@ def get_all_revisions_files( experiment_type: tp.Optional[tp.Type["exp_u.VersionExperiment"]] = None, report_type: tp.Optional[tp.Type[BaseReport]] = None, file_name_filter: tp.Callable[[str], bool] = lambda x: False, - only_newest: bool = True + only_newest: bool = True, + config_id: tp.Optional[int] = None ) -> tp.List[ReportFilepath]: """ Find all file paths to revision files. @@ -188,8 +198,13 @@ def get_all_revisions_files( a list of file paths to correctly processed revision files """ return __get_files_with_status( - project_name, list(FileStatusExtension.get_physical_file_statuses()), - experiment_type, report_type, file_name_filter, only_newest + project_name=project_name, + file_statuses=list(FileStatusExtension.get_physical_file_statuses()), + experiment_type=experiment_type, + report_type=report_type, + file_name_filter=file_name_filter, + only_newest=only_newest, + config_id=config_id ) @@ -198,7 +213,8 @@ def get_processed_revisions_files( experiment_type: tp.Optional[tp.Type["exp_u.VersionExperiment"]] = None, report_type: tp.Optional[tp.Type[BaseReport]] = None, file_name_filter: tp.Callable[[str], bool] = lambda x: False, - only_newest: bool = True + only_newest: bool = True, + config_id: tp.Optional[int] = None ) -> tp.List[ReportFilepath]: """ Find all file paths to correctly processed revision files. @@ -218,8 +234,13 @@ def get_processed_revisions_files( a list of file paths to correctly processed revision files """ return __get_files_with_status( - project_name, [FileStatusExtension.SUCCESS], experiment_type, - report_type, file_name_filter, only_newest + project_name=project_name, + file_statuses=[FileStatusExtension.SUCCESS], + experiment_type=experiment_type, + report_type=report_type, + file_name_filter=file_name_filter, + only_newest=only_newest, + config_id=config_id ) @@ -228,7 +249,8 @@ def get_failed_revisions_files( experiment_type: tp.Optional[tp.Type["exp_u.VersionExperiment"]] = None, report_type: tp.Optional[tp.Type[BaseReport]] = None, file_name_filter: tp.Callable[[str], bool] = lambda x: False, - only_newest: bool = True + only_newest: bool = True, + config_id: tp.Optional[int] = None ) -> tp.List[ReportFilepath]: """ Find all file paths to failed revision files. @@ -248,9 +270,15 @@ def get_failed_revisions_files( a list of file paths to failed revision files """ return __get_files_with_status( - project_name, - [FileStatusExtension.FAILED, FileStatusExtension.COMPILE_ERROR], - experiment_type, report_type, file_name_filter, only_newest + project_name=project_name, + file_statuses=[ + FileStatusExtension.FAILED, FileStatusExtension.COMPILE_ERROR + ], + experiment_type=experiment_type, + report_type=report_type, + file_name_filter=file_name_filter, + only_newest=only_newest, + config_id=config_id ) From 9da46bc546053fbed0eca6f95626d8a3fb4dbab6 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Wed, 29 Nov 2023 11:38:24 +0100 Subject: [PATCH 2/3] Extends TEF report and fixes error handling in perf (#867) --- .../varats/report/linux_perf_report.py | 4 +- varats-core/varats/report/tef_report.py | 46 +++++++++++++++++-- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/varats-core/varats/report/linux_perf_report.py b/varats-core/varats/report/linux_perf_report.py index ef0eacdea..10e661dff 100644 --- a/varats-core/varats/report/linux_perf_report.py +++ b/varats-core/varats/report/linux_perf_report.py @@ -65,9 +65,9 @@ def __parse_ctx_switches(line: str) -> int: return int(line.split(" ")[0].replace(",", "")) @staticmethod - def __parse_branch_misses(line: str) -> tp.Optional[int]: + def __parse_branch_misses(line: str) -> int: if line.startswith(""): - return None + return np.NaN return int(line.split(" ")[0].replace(",", "")) @property diff --git a/varats-core/varats/report/tef_report.py b/varats-core/varats/report/tef_report.py index 44bfb8e36..9fcc6ee28 100644 --- a/varats-core/varats/report/tef_report.py +++ b/varats-core/varats/report/tef_report.py @@ -1,6 +1,7 @@ """Report module to create and handle trace event format files, e.g., created with chrome tracing.""" +import logging import re import typing as tp from enum import Enum @@ -11,6 +12,8 @@ from varats.experiment.workload_util import WorkloadSpecificReportAggregate from varats.report.report import BaseReport, ReportAggregate +LOG = logging.getLogger(__name__) + class TraceEventType(Enum): """Enum to represent the different event types of trace format events, @@ -49,6 +52,8 @@ class TraceEvent(): """Represents a trace event that was captured during the analysis of a target program.""" + __uuid: int + def __init__( self, json_trace_event: tp.Dict[str, tp.Any], name_id: int, name_id_mapper: 'TEFReport.NameIDMapper' @@ -63,6 +68,14 @@ def __init__( self.__pid = int(json_trace_event["pid"]) self.__tid = int(json_trace_event["tid"]) + if "UUID" in json_trace_event: + self.__uuid = int(json_trace_event["UUID"]) + elif "ID" in json_trace_event: + self.__uuid = int(json_trace_event["ID"]) + else: + LOG.critical("Could not parse UUID/ID from trace event") + self.__uuid: int = 0 + @property def name(self) -> str: return self.__name_id_mapper.infer_name(self.__name_id) @@ -87,9 +100,14 @@ def pid(self) -> int: def tid(self) -> int: return self.__tid + @property + def uuid(self) -> int: + return self.__uuid + def __str__(self) -> str: return f"""{{ name: {self.name} + uuid: {self.uuid} cat: {self.category} ph: {self.event_type} ts: {self.timestamp} @@ -99,7 +117,7 @@ def __str__(self) -> str: """ def __repr__(self) -> str: - return str(self) + return f"{{ name={self.name}, uuid={self.uuid} }}" class TEFReport(BaseReport, shorthand="TEF", file_type="json"): @@ -114,7 +132,11 @@ def infer_name(self, name_id: int) -> str: def __init__(self, path: Path) -> None: super().__init__(path) self.__name_id_mapper: TEFReport.NameIDMapper = TEFReport.NameIDMapper() - self._parse_json() + try: + self._parse_json() + except Exception as e: + print(f"Could not parse file: {self.path}") + raise e # Parsing stackFrames is currently not implemented # x = data["stackFrames"] @@ -132,8 +154,26 @@ def stack_frames(self) -> None: "Stack frame parsing is currently not implemented!" ) + def _patch_errors_from_file(self) -> None: + with open(self.path, "r") as f: + data = f.read() + + with open(self.path, "w") as f: + remove_lost_events = re.compile('Lost \d+ events') + for line in data.splitlines(): + if "Lost" in line: + LOG.error( + "Events where lost during tracing, patching json file." + ) + line = remove_lost_events.sub("", line) + + f.write(line) + def _parse_json(self) -> None: - trace_events: tp.List[TraceEvent] = list() + trace_events: tp.List[TraceEvent] = [] + + self._patch_errors_from_file() + with open(self.path, "rb") as f: parser = ijson.parse(f) trace_event: tp.Dict[str, str] = {} From 06076ac248e51a5d9c873dcada1fd64c1294a3e3 Mon Sep 17 00:00:00 2001 From: Florian Sattler Date: Wed, 29 Nov 2023 11:48:18 +0100 Subject: [PATCH 3/3] Updates plotting libs (#866) --- requirements.txt | 4 ++-- varats/setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1a3860088..d1b506d98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ ijson>=3.1.4 Jinja2>=3.1.2 jupyter>=1.0.0 kaleido>=0.2.1 -matplotlib>=3.7.1 +matplotlib>=3.8.2 networkx>=3.0 numpy>=1.24.2 packaging>=20.1 @@ -28,7 +28,7 @@ PyYAML>=6.0 requests>=2.28.2 rich>=12.6 scikit-learn>=1.2.2 -seaborn>=0.12.2 +seaborn>=0.13.0 tabulate>=0.9 types-PyYAML types-requests diff --git a/varats/setup.py b/varats/setup.py index f8457097d..1986b251d 100644 --- a/varats/setup.py +++ b/varats/setup.py @@ -23,7 +23,7 @@ "graphviz>=0.14.2", "Jinja2>=3.1.2", "kaleido>=0.2.1", - "matplotlib>=3.7.1", + "matplotlib>=3.8.2", "networkx>=3.0", "numpy>=1.24.2", "packaging>=20.1", @@ -40,7 +40,7 @@ "PyYAML>=6.0", "rich>=12.6", "scikit-learn>=1.2.2", - "seaborn>=0.12.2", + "seaborn>=0.13.0", "tabulate>=0.9", "varats-core>=13.0.5", "wllvm>=1.3.1",