From d7a6723837647a6271ef84b96396560945d156eb Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Wed, 6 Nov 2024 14:45:12 +0100 Subject: [PATCH] fix: more parsing bugs fixed --- src/sysdiagnose/__init__.py | 18 ++++++----- .../parsers/security_sysdiagnose.py | 30 ++++++++++++------- src/sysdiagnose/parsers/transparency.py | 8 +++-- tests/test_parsers_security_sysdiagnose.py | 6 ++-- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/src/sysdiagnose/__init__.py b/src/sysdiagnose/__init__.py index 2e1d853..927f22d 100644 --- a/src/sysdiagnose/__init__.py +++ b/src/sysdiagnose/__init__.py @@ -124,6 +124,8 @@ def create_case(self, sysdiagnose_file: str, force: bool = False, case_id: bool 'tags': [] } + print(f"Case ID: {str(case['case_id'])}") + # create case folder case_data_folder = self.config.get_case_data_folder(str(case['case_id'])) os.makedirs(case_data_folder, exist_ok=True) @@ -149,12 +151,15 @@ def create_case(self, sysdiagnose_file: str, force: bool = False, case_id: bool remotectl_dumpstate_parser = remotectl_dumpstate.RemotectlDumpstateParser(self.config, case_id) remotectl_dumpstate_json = remotectl_dumpstate_parser.get_result() if 'error' not in remotectl_dumpstate_json: - try: - case['serial_number'] = remotectl_dumpstate_json['Local device']['Properties']['SerialNumber'] - case['unique_device_id'] = remotectl_dumpstate_json['Local device']['Properties']['UniqueDeviceID'] - case['ios_version'] = remotectl_dumpstate_json['Local device']['Properties']['OSVersion'] - except (KeyError, TypeError): - logger.warning("WARNING: Could not parse remotectl_dumpstate, and therefore extract serial numbers.", exc_info=True) + if 'Local device' in remotectl_dumpstate_json: + try: + case['serial_number'] = remotectl_dumpstate_json['Local device']['Properties']['SerialNumber'] + case['unique_device_id'] = remotectl_dumpstate_json['Local device']['Properties']['UniqueDeviceID'] + case['ios_version'] = remotectl_dumpstate_json['Local device']['Properties']['OSVersion'] + except (KeyError, TypeError): + logger.warning("WARNING: Could not parse remotectl_dumpstate, and therefore extract serial numbers.", exc_info=True) + else: + logger.info("WARNING: remotectl_dumpstate does not contain a Local device section.") try: case['date'] = remotectl_dumpstate_parser.sysdiagnose_creation_datetime.isoformat(timespec='microseconds') @@ -173,7 +178,6 @@ def create_case(self, sysdiagnose_file: str, force: bool = False, case_id: bool finally: fcntl.flock(f, fcntl.LOCK_UN) # release lock whatever happens - print(f"Case ID: {str(case['case_id'])}") print(f"Sysdiagnose file has been processed: {sysdiagnose_file}") return case diff --git a/src/sysdiagnose/parsers/security_sysdiagnose.py b/src/sysdiagnose/parsers/security_sysdiagnose.py index 92d76d3..c93b38b 100644 --- a/src/sysdiagnose/parsers/security_sysdiagnose.py +++ b/src/sysdiagnose/parsers/security_sysdiagnose.py @@ -1,3 +1,4 @@ +import glob import os import re from sysdiagnose.utils.base import BaseParserInterface, logger @@ -6,21 +7,23 @@ class SecuritySysdiagnoseParser(BaseParserInterface): description = "Parsing security-sysdiagnose.txt file containing keychain information" - format = 'jsonl' + format = 'json' def __init__(self, config: dict, case_id: str): super().__init__(__file__, config, case_id) def get_log_files(self) -> list: - """ - Get the list of log files to be parsed - """ - log_files = [ + log_files_globs = [ "security-sysdiagnose.txt" ] - return [os.path.join(self.case_data_subfolder, log_files) for log_files in log_files] - - def execute(self) -> list: + log_files = [] + for log_files_glob in log_files_globs: + for item in glob.glob(os.path.join(self.case_data_subfolder, log_files_glob)): + if os.path.getsize(item) > 0: + log_files.append(item) + return log_files + + def execute(self) -> dict: log_files = self.get_log_files() if not log_files: return {'errors': ['No security-sysdiagnose.txt file present']} @@ -173,7 +176,14 @@ def process_buffer_client(buffer: list, json_result: dict): while i < len(buffer): line = buffer[i] match = re.search(r'^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [+-]\d{4}) ([^:]+): (.+?) - Attributes: {(.*)', line) - timestamp = datetime.fromisoformat(match.group(1)) + if match: + timestamp = datetime.fromisoformat(match.group(1)) + else: + match = re.search(r'^(\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2} [AP]M [+-]\d{4}) ([^:]+): (.+?) - Attributes: {(.*)', line) + if match: + timestamp = datetime.strptime(match.group(1), "%Y-%m-%d %I:%M:%S %p %z") + else: + raise ValueError(f"Cannot parse line: {line}") row = { 'timestamp': timestamp.timestamp(), 'datetime': timestamp.isoformat(timespec='microseconds'), @@ -186,7 +196,7 @@ def process_buffer_client(buffer: list, json_result: dict): # while next rows do not start with a date, they are part of the attributes try: - while not re.search(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', buffer[i + 1]): + while not re.search(r'^\d{4}-\d{2}-\d{2} ', buffer[i + 1]): i += 1 attribute_string += buffer[i] except IndexError: diff --git a/src/sysdiagnose/parsers/transparency.py b/src/sysdiagnose/parsers/transparency.py index 9cd5a5f..fcbdc0c 100644 --- a/src/sysdiagnose/parsers/transparency.py +++ b/src/sysdiagnose/parsers/transparency.py @@ -26,8 +26,12 @@ def get_log_files(self) -> list: def execute(self) -> dict: files = self.get_log_files() if not files: - logger.warning("No known transparency.log file found.") + logger.info("No known transparency.log file found.") return {} for file in files: with open(file, 'r') as f: - return json.load(f) + try: + return json.load(f) + except json.decoder.JSONDecodeError: + logger.warning(f"Error parsing {file}") + return {} diff --git a/tests/test_parsers_security_sysdiagnose.py b/tests/test_parsers_security_sysdiagnose.py index b608862..4a68ca7 100644 --- a/tests/test_parsers_security_sysdiagnose.py +++ b/tests/test_parsers_security_sysdiagnose.py @@ -42,12 +42,14 @@ def test_process_buffer_client(self): '2023-05-24 19:55:51 +0000 EventSoftFailure: OTAPKIEvent - Attributes: {product : iPhone OS, build : 19H349, errorDomain : NSOSStatusErrorDomain, modelid : iPhone9,3, errorCode : -67694 }', '2023-05-24 19:57:58 +0000 EventSoftFailure: MitmDetectionEvent - Attributes: {product : iPhone OS, build : 19H349, overallScore : 0, timeSinceLastReset : 127, rootUsages : (', ' foo, bar ', - '), errorDomain : MITMErrorDomain, modelid : iPhone9,3, errorCode : 0 }' + '), errorDomain : MITMErrorDomain, modelid : iPhone9,3, errorCode : 0 }', + '2024-09-06 9:45:15 AM +0000 EventHardFailure: CloudServicesSetConfirmedManifest - Attributes: {product : macOS, build : 22H121, errorDomain : NSOSStatusErrorDomain, modelid : MacBookPro14,3, errorCode : -25308 }', ] expected_output = { 'events': [ {'datetime': '2023-05-24T19:55:51.000000+00:00', 'timestamp': 1684958151.0, 'section': 'client_trust', 'result': 'EventSoftFailure', 'event': 'OTAPKIEvent', 'attributes': {'product': 'iPhone OS', 'build': '19H349', 'errorDomain': 'NSOSStatusErrorDomain', 'modelid': 'iPhone9,3', 'errorCode': '-67694'}}, - {'datetime': '2023-05-24T19:57:58.000000+00:00', 'timestamp': 1684958278.0, 'section': 'client_trust', 'result': 'EventSoftFailure', 'event': 'MitmDetectionEvent', 'attributes': {'product': 'iPhone OS', 'build': '19H349', 'overallScore': '0', 'timeSinceLastReset': '127', 'rootUsages': '( foo, bar )', 'errorDomain': 'MITMErrorDomain', 'modelid': 'iPhone9,3', 'errorCode': '0'}} + {'datetime': '2023-05-24T19:57:58.000000+00:00', 'timestamp': 1684958278.0, 'section': 'client_trust', 'result': 'EventSoftFailure', 'event': 'MitmDetectionEvent', 'attributes': {'product': 'iPhone OS', 'build': '19H349', 'overallScore': '0', 'timeSinceLastReset': '127', 'rootUsages': '( foo, bar )', 'errorDomain': 'MITMErrorDomain', 'modelid': 'iPhone9,3', 'errorCode': '0'}}, + {'datetime': '2024-09-06T09:45:15.000000+00:00', 'timestamp': 1725615915.0, 'section': 'client_trust', 'result': 'EventHardFailure', 'event': 'CloudServicesSetConfirmedManifest', 'attributes': {'product': 'macOS', 'build': '22H121', 'errorDomain': 'NSOSStatusErrorDomain', 'modelid': 'MacBookPro14,3', 'errorCode': '-25308'}} ] } result = {'events': []}