chg: [parser] multilinelog makes clean timestamp

EC-DIGIT-CSIRC · Jun 12, 2024 · 5cadcce · 5cadcce
1 parent ef81a77
commit 5cadcce
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 12 deletions.
diff --git a/tests/test_multilinelog.py b/tests/test_multilinelog.py
@@ -26,7 +26,8 @@ def test_multilinelog_plist(self):
 
 '''
         expected_result = {
-            'timestamp': '2023-05-24 12:58:04',
+            'timestamp': 1684925884000000,
+            'datetime': '2023-05-24T12:58:04+00:00',
             'loglevel': 'debug',
             'hexID': '0x16bf9b000',
             'event_type': 'MobileActivationDaemon handleActivationInfoWithSession:activationSignature:completionBlock:',
@@ -61,7 +62,8 @@ def test_multilinelog_curlybrackets(self):
 } (Error Domain=MCMErrorDomain Code=29 "Invalid metadata-URLs should match: /private/var/mobile/Containers/Data/Application/0984009B-81D1-4F7F-BDBD-261E22059155 : /private/var/mobile/Containers/Data/VPNPlugin/0984009B-81D1-4F7F-BDBD-261E22059155" UserInfo={SourceFileLine=370, NSLocalizedDescription=Invalid metadata-URLs should match: /private/var/mobile/Containers/Data/Application/0984009B-81D1-4F7F-BDBD-261E22059155 : /private/var/mobile/Containers/Data/VPNPlugin/0984009B-81D1-4F7F-BDBD-261E22059155, FunctionName=+[MCMMetadata _readAndValidateMetadataInDictionary:containerURL:forUserIdentity:containerClass:checkClassPath:fsNode:transient:error:]})
 '''
         expected_result = {
-            'timestamp': '2023-05-24 13:05:36',
+            'timestamp': 1684926336000000,
+            'datetime': '2023-05-24T13:05:36+00:00',
             'loglevel': 'err',
             'hexID': '0x16be43000',
             'msg': '+[MCMMetadata readAndValidateMetadataAtFileUrl:forUserIdentity:containerClass:checkClassPath:transient:error:]: 199: Failed to validate metadata at URL [file:///private/var/mobile/Containers/Data/Application/0984009B-81D1-4F7F-BDBD-261E22059155/.com.apple.mobile_container_manager.metadata.plist]: {\n    MCMMetadataActiveDPClass = 0;\n    MCMMetadataContentClass = 2;\n    MCMMetadataIdentifier = "com.apple.VoiceMemos";\n    MCMMetadataInfo =     {\n        "com.apple.MobileInstallation.ContentProtectionClass" = 0;\n    };\n    MCMMetadataSchemaVersion = 1;\n    MCMMetadataUUID = "12036663-1F3A-45B3-A34C-402D5BB7D4FB";\n    MCMMetadataUserIdentity =     {\n        personaUniqueString = "83CB8039-725D-4462-84C2-7F79F0A6EFB3";\n        posixGID = 501;\n        posixUID = 501;\n        type = 0;\n        version = 2;\n    };\n    MCMMetadataVersion = 6;\n} (Error Domain=MCMErrorDomain Code=29 "Invalid metadata-URLs should match: /private/var/mobile/Containers/Data/Application/0984009B-81D1-4F7F-BDBD-261E22059155 : /private/var/mobile/Containers/Data/VPNPlugin/0984009B-81D1-4F7F-BDBD-261E22059155" UserInfo={SourceFileLine=370, NSLocalizedDescription=Invalid metadata-URLs should match: /private/var/mobile/Containers/Data/Application/0984009B-81D1-4F7F-BDBD-261E22059155 : /private/var/mobile/Containers/Data/VPNPlugin/0984009B-81D1-4F7F-BDBD-261E22059155, FunctionName=+[MCMMetadata _readAndValidateMetadataInDictionary:containerURL:forUserIdentity:containerClass:checkClassPath:fsNode:transient:error:]})'}
@@ -71,7 +73,8 @@ def test_multilinelog_curlybrackets(self):
     def test_multilinelog_simple_1(self):
         s = '''Wed May 24 12:55:37 2023 [72] <notice> (0x16afb3000) -[MCMClientConnection _regenerateAllSystemContainerPaths]: Rolling system container directory UUIDs on disk'''
         expected_result = {
-            'timestamp': '2023-05-24 12:55:37',
+            'timestamp': 1684925737000000,
+            'datetime': '2023-05-24T12:55:37+00:00',
             'loglevel': 'notice',
             'hexID': '0x16afb3000',
             'event_type': 'MCMClientConnection _regenerateAllSystemContainerPaths',
@@ -82,7 +85,8 @@ def test_multilinelog_simple_1(self):
     def test_mutlinelog_simple_2(self):
         s = '''Wed May 24 13:05:30 2023 [72] <notice> (0x16be43000) _containermanagerd_init_block_invoke: containermanagerd first boot cleanup complete'''
         expected_result = {
-            'timestamp': '2023-05-24 13:05:30',
+            'timestamp': 1684926330000000,
+            'datetime': '2023-05-24T13:05:30+00:00',
             'loglevel': 'notice',
             'hexID': '0x16be43000',
             'msg': '_containermanagerd_init_block_invoke: containermanagerd first boot cleanup complete'}
@@ -93,12 +97,14 @@ def test_multilinelog_simple_multiplelines(self):
         s = '''Wed May 24 13:05:30 2023 [72] <notice> (0x16be43000) _containermanagerd_init_block_invoke: containermanagerd first boot cleanup complete
 Wed May 24 12:55:37 2023 [72] <notice> (0x16afb3000) -[MCMClientConnection _regenerateAllSystemContainerPaths]: Rolling system container directory UUIDs on disk'''
         expected_result_0 = {
-            'timestamp': '2023-05-24 13:05:30',
+            'timestamp': 1684926330000000,
+            'datetime': '2023-05-24T13:05:30+00:00',
             'loglevel': 'notice',
             'hexID': '0x16be43000',
             'msg': '_containermanagerd_init_block_invoke: containermanagerd first boot cleanup complete'}
         expected_result_1 = {
-            'timestamp': '2023-05-24 12:55:37',
+            'timestamp': 1684925737000000,
+            'datetime': '2023-05-24T12:55:37+00:00',
             'loglevel': 'notice',
             'hexID': '0x16afb3000',
             'event_type': 'MCMClientConnection _regenerateAllSystemContainerPaths',
@@ -132,13 +138,15 @@ def test_multilinelog_keyvalue(self):
 Wed May 24 13:08:13 2023 [135] <debug> (0x16f1db000) MA: main: soc_generation: H9
 Wed May 24 13:08:13 2023 [135] <debug> (0x16f1db000) MA: main: ____________________________________________________________________'''
         expected_result_0 = {
-            'timestamp': '2023-05-24 12:55:37',
+            'timestamp': 1684925737000000,
+            'datetime': '2023-05-24T12:55:37+00:00',
             'loglevel': 'notice',
             'hexID': '0x16afb3000',
             'event_type': 'MCMClientConnection _regenerateAllSystemContainerPaths',
             'msg': 'Rolling system container directory UUIDs on disk'}
         expected_result_1 = {
-            'timestamp': '2023-05-24 13:08:13',
+            'timestamp': 1684926493000000,
+            'datetime': '2023-05-24T13:08:13+00:00',
             'loglevel': 'debug',
             'hexID': '0x16f1db000',
             'msg': 'MA: main: ____________________ Mobile Activation Startup _____________________',
@@ -165,12 +173,14 @@ def test_multilinelog_keyvalue_onlyend(self):
         s = '''Sat Feb 18 09:48:38 2023 [2695] <debug> (0x16dc37000) MA: main: ____________________________________________________________________
 Sat Feb 18 09:48:39 2023 [2695] <debug> (0x16dc37000) MA: dealwith_activation: Activation State: Activated'''
         expected_result_0 = {
-            'timestamp': '2023-02-18 09:48:38',
+            'timestamp': 1676710118000000,
+            'datetime': '2023-02-18T09:48:38+00:00',
             'loglevel': 'debug',
             'hexID': '0x16dc37000',
             'msg': 'MA: main: ____________________________________________________________________'}
         expected_result_1 = {
-            'timestamp': '2023-02-18 09:48:39',
+            'timestamp': 1676710119000000,
+            'datetime': '2023-02-18T09:48:39+00:00',
             'loglevel': 'debug',
             'hexID': '0x16dc37000',
             'msg': 'MA: dealwith_activation: Activation State: Activated'}
@@ -181,7 +191,8 @@ def test_multilinelog_keyvalue_onlyend(self):
     def test_multilinelog_keyvalue_onlystart(self):
         s = '''Fri Dec  2 11:32:19 2022 [84816] <debug> (0x16afff000) MA: main: ____________________ Mobile Activation Startup _____________________'''
         expected_result = {
-            'timestamp': '2022-12-02 11:32:19',
+            'timestamp': 1669977139000000,
+            'datetime': '2022-12-02T11:32:19+00:00',
             'loglevel': 'debug',
             'hexID': '0x16afff000',
             'msg': 'MA: main: ____________________ Mobile Activation Startup _____________________'}

diff --git a/utils/multilinelog.py b/utils/multilinelog.py
@@ -1,6 +1,7 @@
 import re
 import io
 import utils.misc as misc
+from datetime import datetime
 
 
 def extract_from_file(fname):
@@ -77,7 +78,9 @@ def build_from_logentry(line):
         weekday, month, day, time, year = (str.split(timestamp[:24]))
         day = day_converter(day)
         month = month_converter(month)
-        entry['timestamp'] = str(year) + '-' + str(month) + '-' + str(day) + ' ' + str(time)
+        timestamp = datetime.strptime(f"{year}-{month}-{day} {time}", "%Y-%m-%d %H:%M:%S")
+        entry['timestamp'] = int(timestamp.timestamp() * 1000000)
+        entry['datetime'] = timestamp.strftime("%Y-%m-%dT%H:%M:%S+00:00")
 
         # log level
         loglevelregex = re.search(r"\<(.*?)\>", line)