fix: more parsing bugs fixed

src/sysdiagnose/__init__.py

@@ -124,6 +124,8 @@ def create_case(self, sysdiagnose_file: str, force: bool = False, case_id: bool
                 'tags': []
             }
 
+        print(f"Case ID: {str(case['case_id'])}")
+
         # create case folder
         case_data_folder = self.config.get_case_data_folder(str(case['case_id']))
         os.makedirs(case_data_folder, exist_ok=True)
@@ -149,12 +151,15 @@ def create_case(self, sysdiagnose_file: str, force: bool = False, case_id: bool
         remotectl_dumpstate_parser = remotectl_dumpstate.RemotectlDumpstateParser(self.config, case_id)
         remotectl_dumpstate_json = remotectl_dumpstate_parser.get_result()
         if 'error' not in remotectl_dumpstate_json:
-            try:
-                case['serial_number'] = remotectl_dumpstate_json['Local device']['Properties']['SerialNumber']
-                case['unique_device_id'] = remotectl_dumpstate_json['Local device']['Properties']['UniqueDeviceID']
-                case['ios_version'] = remotectl_dumpstate_json['Local device']['Properties']['OSVersion']
-            except (KeyError, TypeError):
-                logger.warning("WARNING: Could not parse remotectl_dumpstate, and therefore extract serial numbers.", exc_info=True)
+            if 'Local device' in remotectl_dumpstate_json:
+                try:
+                    case['serial_number'] = remotectl_dumpstate_json['Local device']['Properties']['SerialNumber']
+                    case['unique_device_id'] = remotectl_dumpstate_json['Local device']['Properties']['UniqueDeviceID']
+                    case['ios_version'] = remotectl_dumpstate_json['Local device']['Properties']['OSVersion']
+                except (KeyError, TypeError):
+                    logger.warning("WARNING: Could not parse remotectl_dumpstate, and therefore extract serial numbers.", exc_info=True)
+            else:
+                logger.info("WARNING: remotectl_dumpstate does not contain a Local device section.")
 
         try:
             case['date'] = remotectl_dumpstate_parser.sysdiagnose_creation_datetime.isoformat(timespec='microseconds')
@@ -173,7 +178,6 @@ def create_case(self, sysdiagnose_file: str, force: bool = False, case_id: bool
             finally:
                 fcntl.flock(f, fcntl.LOCK_UN)        # release lock whatever happens
 
-        print(f"Case ID: {str(case['case_id'])}")
         print(f"Sysdiagnose file has been processed: {sysdiagnose_file}")
         return case
 

src/sysdiagnose/parsers/security_sysdiagnose.py

@@ -1,3 +1,4 @@
+import glob
 import os
 import re
 from sysdiagnose.utils.base import BaseParserInterface, logger
@@ -6,21 +7,23 @@
 
 class SecuritySysdiagnoseParser(BaseParserInterface):
     description = "Parsing security-sysdiagnose.txt file containing keychain information"
-    format = 'jsonl'
+    format = 'json'
 
     def __init__(self, config: dict, case_id: str):
         super().__init__(__file__, config, case_id)
 
     def get_log_files(self) -> list:
-        """
-        Get the list of log files to be parsed
-        """
-        log_files = [
+        log_files_globs = [
             "security-sysdiagnose.txt"
         ]
-        return [os.path.join(self.case_data_subfolder, log_files) for log_files in log_files]
-
-    def execute(self) -> list:
+        log_files = []
+        for log_files_glob in log_files_globs:
+            for item in glob.glob(os.path.join(self.case_data_subfolder, log_files_glob)):
+                if os.path.getsize(item) > 0:
+                    log_files.append(item)
+        return log_files
+
+    def execute(self) -> dict:
         log_files = self.get_log_files()
         if not log_files:
             return {'errors': ['No security-sysdiagnose.txt file present']}
@@ -173,7 +176,14 @@ def process_buffer_client(buffer: list, json_result: dict):
         while i < len(buffer):
             line = buffer[i]
             match = re.search(r'^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [+-]\d{4}) ([^:]+): (.+?) - Attributes: {(.*)', line)
-            timestamp = datetime.fromisoformat(match.group(1))
+            if match:
+                timestamp = datetime.fromisoformat(match.group(1))
+            else:
+                match = re.search(r'^(\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2} [AP]M [+-]\d{4}) ([^:]+): (.+?) - Attributes: {(.*)', line)
+                if match:
+                    timestamp = datetime.strptime(match.group(1), "%Y-%m-%d %I:%M:%S %p %z")
+                else:
+                    raise ValueError(f"Cannot parse line: {line}")
             row = {
                 'timestamp': timestamp.timestamp(),
                 'datetime': timestamp.isoformat(timespec='microseconds'),
@@ -186,7 +196,7 @@ def process_buffer_client(buffer: list, json_result: dict):
 
             # while next rows do not start with a date, they are part of the attributes
             try:
-                while not re.search(r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', buffer[i + 1]):
+                while not re.search(r'^\d{4}-\d{2}-\d{2} ', buffer[i + 1]):
                     i += 1
                     attribute_string += buffer[i]
             except IndexError:

src/sysdiagnose/parsers/transparency.py

@@ -26,8 +26,12 @@ def get_log_files(self) -> list:
     def execute(self) -> dict:
         files = self.get_log_files()
         if not files:
-            logger.warning("No known transparency.log file found.")
+            logger.info("No known transparency.log file found.")
             return {}
         for file in files:
             with open(file, 'r') as f:
-                return json.load(f)
+                try:
+                    return json.load(f)
+                except json.decoder.JSONDecodeError:
+                    logger.warning(f"Error parsing {file}")
+                    return {}

tests/test_parsers_security_sysdiagnose.py

@@ -42,12 +42,14 @@ def test_process_buffer_client(self):
             '2023-05-24 19:55:51 +0000 EventSoftFailure: OTAPKIEvent - Attributes: {product : iPhone OS, build : 19H349, errorDomain : NSOSStatusErrorDomain, modelid : iPhone9,3, errorCode : -67694 }',
             '2023-05-24 19:57:58 +0000 EventSoftFailure: MitmDetectionEvent - Attributes: {product : iPhone OS, build : 19H349, overallScore : 0, timeSinceLastReset : 127, rootUsages : (',
             ' foo, bar ',
-            '), errorDomain : MITMErrorDomain, modelid : iPhone9,3, errorCode : 0 }'
+            '), errorDomain : MITMErrorDomain, modelid : iPhone9,3, errorCode : 0 }',
+            '2024-09-06 9:45:15 AM +0000 EventHardFailure: CloudServicesSetConfirmedManifest - Attributes: {product : macOS, build : 22H121, errorDomain : NSOSStatusErrorDomain, modelid : MacBookPro14,3, errorCode : -25308 }',
         ]
         expected_output = {
             'events': [
                 {'datetime': '2023-05-24T19:55:51.000000+00:00', 'timestamp': 1684958151.0, 'section': 'client_trust', 'result': 'EventSoftFailure', 'event': 'OTAPKIEvent', 'attributes': {'product': 'iPhone OS', 'build': '19H349', 'errorDomain': 'NSOSStatusErrorDomain', 'modelid': 'iPhone9,3', 'errorCode': '-67694'}},
-                {'datetime': '2023-05-24T19:57:58.000000+00:00', 'timestamp': 1684958278.0, 'section': 'client_trust', 'result': 'EventSoftFailure', 'event': 'MitmDetectionEvent', 'attributes': {'product': 'iPhone OS', 'build': '19H349', 'overallScore': '0', 'timeSinceLastReset': '127', 'rootUsages': '( foo, bar )', 'errorDomain': 'MITMErrorDomain', 'modelid': 'iPhone9,3', 'errorCode': '0'}}
+                {'datetime': '2023-05-24T19:57:58.000000+00:00', 'timestamp': 1684958278.0, 'section': 'client_trust', 'result': 'EventSoftFailure', 'event': 'MitmDetectionEvent', 'attributes': {'product': 'iPhone OS', 'build': '19H349', 'overallScore': '0', 'timeSinceLastReset': '127', 'rootUsages': '( foo, bar )', 'errorDomain': 'MITMErrorDomain', 'modelid': 'iPhone9,3', 'errorCode': '0'}},
+                {'datetime': '2024-09-06T09:45:15.000000+00:00', 'timestamp': 1725615915.0, 'section': 'client_trust', 'result': 'EventHardFailure', 'event': 'CloudServicesSetConfirmedManifest', 'attributes': {'product': 'macOS', 'build': '22H121', 'errorDomain': 'NSOSStatusErrorDomain', 'modelid': 'MacBookPro14,3', 'errorCode': '-25308'}}
             ]
         }
         result = {'events': []}