diff --git a/parsers/spindumpnosymbols.py b/parsers/spindumpnosymbols.py index 75680d0..6abcb04 100644 --- a/parsers/spindumpnosymbols.py +++ b/parsers/spindumpnosymbols.py @@ -16,14 +16,13 @@ -v --version Show version. """ -import sys -from optparse import OptionParser -import plistlib -import json from docopt import docopt -from tabulate import tabulate +import glob +import json +import os import re -import pprint +import sys + # ----- definition for parsing.py script -----# # ----- DO NOT DELETE ----# @@ -35,6 +34,17 @@ # --------------------------------------------# +def get_log_files(log_root_path: str) -> list: + log_files_globs = [ + 'spindump-nosymbols.txt' + ] + log_files = [] + for log_files_glob in log_files_globs: + log_files.extend(glob.glob(os.path.join(log_root_path, log_files_glob))) + + return log_files + + def parsespindumpNS(file): with open(file, 'r') as f_in: # init section @@ -167,7 +177,7 @@ def parse_thread(data): output["loaded"] = [] for line in data[1:]: - loaded={} + loaded = {} if "+" in line: loaded["library"] = line.split("(", 1)[1].split("+", 1)[0].strip() loaded["int"] = line.split("(", 1)[1].split("+", 1)[1].split(")", 1)[0].strip() @@ -179,7 +189,7 @@ def parse_thread(data): def parse_images(data): - images=[] + images = [] for line in data: image = {} if line.strip() is not None: @@ -203,12 +213,12 @@ def main(): arguments = docopt(__doc__, version='parser for networkextension.plist v0.1') - ### test + # ## test if arguments['-i']: # Output is good enough, just print print(json.dumps(parsespindumpNS(arguments['']), indent=4)) sys.exit() - ### test + # ## test return 0 diff --git a/parsers/swcutil.py b/parsers/swcutil.py index 3edbc91..84a66d4 100644 --- a/parsers/swcutil.py +++ b/parsers/swcutil.py @@ -16,13 +16,10 @@ -v --version Show version. """ -import sys -from optparse import OptionParser -import plistlib -import json from docopt import docopt -from tabulate import tabulate - +import glob +import os +import sys # ----- definition for parsing.py script -----# # ----- DO NOT DELETE ----# @@ -34,6 +31,17 @@ # --------------------------------------------# +def get_log_files(log_root_path: str) -> list: + log_files_globs = [ + 'swcutil_show.txt' + ] + log_files = [] + for log_files_glob in log_files_globs: + log_files.extend(glob.glob(os.path.join(log_root_path, log_files_glob))) + + return log_files + + def parseswcutil(file): with open(file, 'r') as f_in: # init section @@ -115,7 +123,7 @@ def main(): arguments = docopt(__doc__, version='parser for networkextension.plist v0.1') - ### test + # ## test if arguments['-i']: # Output is good enough, just print with open(arguments[''], 'r') as f_in: @@ -123,7 +131,7 @@ def main(): print(line.strip()) # parseswcutil(arguments['']) sys.exit() - ### test + # ## test return 0 diff --git a/parsers/sys.py b/parsers/sys.py index 89f30cb..37c9f79 100644 --- a/parsers/sys.py +++ b/parsers/sys.py @@ -9,6 +9,8 @@ import sys from optparse import OptionParser import plistlib +import os +import glob version_string = "sysdiagnose-sys.py v2019-05-10 Version 2.0" @@ -22,6 +24,17 @@ # --------------------------------------------# +def get_log_files(log_root_path: str) -> list: + log_files_globs = [ + 'logs/SystemVersion/SystemVersion.plist' + ] + log_files = [] + for log_files_glob in log_files_globs: + log_files.extend(glob.glob(os.path.join(log_root_path, log_files_glob))) + + return log_files + + # --------------------------------------------------------------------------- # # XXX FIXME: this is obviously a very generic function and could be generalized and moved to misc/ diff --git a/parsers/taskinfo.py b/parsers/taskinfo.py index 48396ff..fda6b13 100644 --- a/parsers/taskinfo.py +++ b/parsers/taskinfo.py @@ -9,7 +9,8 @@ # import re import sys -import json +import glob +import os from optparse import OptionParser version_string = "sysdiagnose-taskinfo.py v2020-02-07 Version 1.0" @@ -24,6 +25,17 @@ # --------------------------------------------# +def get_log_files(log_root_path: str) -> list: + log_files_globs = [ + 'taskinfo.txt' + ] + log_files = [] + for log_files_glob in log_files_globs: + log_files.extend(glob.glob(os.path.join(log_root_path, log_files_glob))) + + return log_files + + # --------------------------------------------------------------------------- # def get_num_tasks(filename, ios_version=13): """ @@ -264,7 +276,7 @@ def get_tasks(filename, ios_version=13): continue fd.close() except Exception as e: - print(f"Could not open {filename}") + print(f"Could not open {filename} for reading. Reason: {str(e)}") return {"numb_tasks": numb_tasks, "tasks": tasks} diff --git a/parsers/wifi_known_networks.py b/parsers/wifi_known_networks.py index c16f952..37127e5 100644 --- a/parsers/wifi_known_networks.py +++ b/parsers/wifi_known_networks.py @@ -13,7 +13,7 @@ import biplist from biplist import Uid, Data from datetime import datetime - +import glob import config @@ -36,6 +36,17 @@ def default(self, obj): # --------------------------------------------# +def get_log_files(log_root_path: str) -> list: + log_files_globs = [ + 'WiFi/com.apple.wifi.known-networks.plist' + ] + log_files = [] + for log_files_glob in log_files_globs: + log_files.extend(glob.glob(os.path.join(log_root_path, log_files_glob))) + + return log_files + + # --------------------------------------------------------------------------- # # XXX #26 FIXME: this is obviously a very generic function and could be generalized and moved to misc/ diff --git a/parsers/wifinetworks.py b/parsers/wifinetworks.py index f42203e..aa56ca2 100644 --- a/parsers/wifinetworks.py +++ b/parsers/wifinetworks.py @@ -21,6 +21,7 @@ import glob # sys.path.append('..') # noqa: E402 import misc # noqa: E402 +import os # ----- definition for parsing.py script -----# @@ -31,6 +32,18 @@ # --------------------------------------------------------------------------- # +def get_log_files(log_root_path: str) -> list: + log_files_globs = [ + 'WiFi/*.plist', + 'WiFi/com.apple.wifi.recent-networks.json' + ] + log_files = [] + for log_files_glob in log_files_globs: + log_files.extend(glob.glob(os.path.join(log_root_path, log_files_glob))) + + return log_files + + def parsewifinetwork(wifi_data: list): output = {} for data in wifi_data: diff --git a/tests/__init__.py b/tests/__init__.py index 6cffb9d..da00c62 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -11,4 +11,4 @@ def setUp(self): # - testdata-private/iOSxx/sysdiagnose_YYYY.MM.DD_HH-MM-SS-SSSS_... # this allows testing locally with more data, while keeping online tests coherent - self.log_root_paths = [name for name in glob.glob('tests/testdata*/*/*') if os.path.isdir(name)] + self.log_root_paths = [name for name in glob.glob('tests/testdata*/**/sysdiagnose_*', recursive=True) if os.path.isdir(name)] diff --git a/tests/test_parsers_spindumpnosymbols.py b/tests/test_parsers_spindumpnosymbols.py index 776370c..05c586a 100644 --- a/tests/test_parsers_spindumpnosymbols.py +++ b/tests/test_parsers_spindumpnosymbols.py @@ -1,18 +1,20 @@ -from parsers.spindumpnosymbols import parsespindumpNS +from parsers.spindumpnosymbols import parsespindumpNS, get_log_files from tests import SysdiagnoseTestCase import unittest class TestParsersSpindumpnosymbols(SysdiagnoseTestCase): - log_path = "tests/testdata/iOS15/sysdiagnose_2023.05.24_13-29-15-0700_iPhone-OS_iPhone_19H349/spindump-nosymbols.txt" - def test_parsespindumpNS(self): - result = parsespindumpNS(self.log_path) - self.assertGreater(len(result), 0) - self.assertTrue('iPhone OS' in result['OS Version']) - self.assertTrue('rootdev' in result['Boot args']) - self.assertGreater(len(result['processes']), 0) + for log_root_path in self.log_root_paths: + files = get_log_files(log_root_path) + self.assertTrue(len(files) > 0) + for file in files: + print(f'Parsing {file}') + result = parsespindumpNS(file) + self.assertGreater(len(result), 0) + self.assertTrue('OS Version' in result) + self.assertGreater(len(result['processes']), 0) if __name__ == '__main__': diff --git a/tests/test_parsers_swcutil.py b/tests/test_parsers_swcutil.py index e8562b7..06b9fc3 100644 --- a/tests/test_parsers_swcutil.py +++ b/tests/test_parsers_swcutil.py @@ -1,24 +1,23 @@ -from parsers.swcutil import parseswcutil +from parsers.swcutil import parseswcutil, get_log_files from tests import SysdiagnoseTestCase import unittest class TestParsersSwcutil(SysdiagnoseTestCase): - - log_path = "tests/testdata/iOS15/sysdiagnose_2023.05.24_13-29-15-0700_iPhone-OS_iPhone_19H349/swcutil_show.txt" - - db_keys = ['Service', 'App ID', 'App Version', 'App PI', 'Domain', 'Patterns', 'User Approval', 'Site/Fmwk Approval', 'Flags'] - def test_parseswcutil(self): - result = parseswcutil(self.log_path) - self.assertGreater(len(result), 0) - self.assertTrue('headers' in result) - self.assertTrue('network' in result) - self.assertTrue('headers' in result) - - self.assertGreater(len(result['memory']), 0) - self.assertGreater(len(result['db']), 0) - self.assertEqual(list(result['db'][0].keys()), self.db_keys) + for log_root_path in self.log_root_paths: + files = get_log_files(log_root_path) + self.assertTrue(len(files) > 0) + for file in files: + print(f'Parsing {file}') + result = parseswcutil(file) + self.assertGreater(len(result), 0) + self.assertTrue('headers' in result) + self.assertTrue('network' in result) + self.assertTrue('headers' in result) + + self.assertGreater(len(result['memory']), 0) + self.assertGreater(len(result['db']), 0) if __name__ == '__main__': diff --git a/tests/test_parsers_sys.py b/tests/test_parsers_sys.py index dde3a5e..4c469b6 100644 --- a/tests/test_parsers_sys.py +++ b/tests/test_parsers_sys.py @@ -1,21 +1,23 @@ -from parsers.sys import getProductInfo +from parsers.sys import getProductInfo, get_log_files from tests import SysdiagnoseTestCase import unittest class TestParsersSys(SysdiagnoseTestCase): - - log_path = "tests/testdata/iOS15/sysdiagnose_2023.05.24_13-29-15-0700_iPhone-OS_iPhone_19H349/logs/SystemVersion/SystemVersion.plist" - productinfo_keys = ['ProductName', 'ProductBuildVersion', 'ProductVersion', 'BuildID', 'SystemImageID'] def test_getProductInfo(self): - result = getProductInfo(self.log_path) - self.assertGreater(len(result), 0) - - self.assertTrue(result.keys() | self.productinfo_keys == result.keys()) # check if the result contains at least the following keys - self.assertTrue('iPhone OS' in result['ProductName']) - self.assertTrue(result['BuildID']) + for log_root_path in self.log_root_paths: + files = get_log_files(log_root_path) + self.assertTrue(len(files) > 0) + for file in files: + print(f'Parsing {file}') + result = getProductInfo(file) + self.assertGreater(len(result), 0) + + self.assertTrue(result.keys() | self.productinfo_keys == result.keys()) # check if the result contains at least the following keys + self.assertTrue('iPhone OS' in result['ProductName']) + self.assertTrue(result['BuildID']) if __name__ == '__main__': diff --git a/tests/test_parsers_taskinfo.py b/tests/test_parsers_taskinfo.py index 160956c..676886d 100644 --- a/tests/test_parsers_taskinfo.py +++ b/tests/test_parsers_taskinfo.py @@ -1,18 +1,20 @@ -from parsers.taskinfo import get_tasks +from parsers.taskinfo import get_tasks, get_log_files from tests import SysdiagnoseTestCase import unittest class TestParsersTaskinfo(SysdiagnoseTestCase): - log_path = "tests/testdata/iOS15/sysdiagnose_2023.05.24_13-29-15-0700_iPhone-OS_iPhone_19H349/taskinfo.txt" - def test_get_tasks(self): - result = get_tasks(self.log_path) - self.assertGreater(len(result), 0) - - self.assertEqual(result['numb_tasks'], 244) - self.assertGreater(len(result['tasks']), 0) + for log_root_path in self.log_root_paths: + files = get_log_files(log_root_path) + self.assertTrue(len(files) > 0) + for file in files: + print(f'Parsing {file}') + result = get_tasks(file) + self.assertGreater(len(result), 0) + self.assertGreater(result['numb_tasks'], 0) + self.assertGreater(len(result['tasks']), 0) if __name__ == '__main__': diff --git a/tests/test_parsers_wifi_known_networks.py b/tests/test_parsers_wifi_known_networks.py index e8683c8..35a490c 100644 --- a/tests/test_parsers_wifi_known_networks.py +++ b/tests/test_parsers_wifi_known_networks.py @@ -1,20 +1,18 @@ -from parsers.wifi_known_networks import getKnownWifiNetworks +from parsers.wifi_known_networks import getKnownWifiNetworks, get_log_files from tests import SysdiagnoseTestCase -import json import unittest class TestParsersWifiKnownNetworks(SysdiagnoseTestCase): - log_path = "tests/testdata/iOS15/sysdiagnose_2023.05.24_13-29-15-0700_iPhone-OS_iPhone_19H349/WiFi/com.apple.wifi.known-networks.plist" - def test_getKnownWifiNetworks(self): - known_networks_str = '{"wifi.network.ssid.FIRSTCONWIFI": {"Moving": false, "Hidden": false, "CaptiveProfile": {"IsWhitelistedCaptiveNetwork": false, "CaptiveNetwork": false, "WhitelistedCaptiveNetworkProbeDate": "2023-05-24 19:57:57.912583"}, "JoinedByUserAt": "2023-05-24 19:57:50.247657", "SSID": "b\'FIRSTCONWIFI\'", "AddedAt": "2023-05-24 19:57:50.248702", "__OSSpecific__": {"BEACON_PROBE_INFO_PER_BSSID_LIST": [{"BSSID": "0:13:37:a9:e7:eb", "OTA_SYSTEM_INFO_SENT": false, "OTA_SYSTEM_INFO_BEACON_ONLY_SENT": true}], "CHANNEL_FLAGS": 10, "prevJoined": "2023-05-24 20:14:36.705016", "BSSID": "0:13:37:a9:e7:eb", "CHANNEL": 11, "AP_MODE": 2, "DiagnosticsBssEnv": 1}, "JoinedBySystemAt": "2023-05-24 20:26:05.388462", "SupportedSecurityTypes": "Open", "BSSList": [{"ChannelFlags": 10, "BSSID": "0:13:37:a9:e7:eb", "Channel": 11, "LastAssociatedAt": "2023-05-24 20:26:05.388434"}], "UpdatedAt": "2023-05-24 20:26:05.670268"}}' - - result = getKnownWifiNetworks([self.log_path]) - self.assertGreater(len(result), 0) - - self.assertEqual(json.loads(known_networks_str), result) + for log_root_path in self.log_root_paths: + files = get_log_files(log_root_path) + self.assertTrue(len(files) > 0) + for file in files: + print(f'Parsing {file}') + result = getKnownWifiNetworks([file]) + self.assertGreater(len(result), 0) if __name__ == '__main__': diff --git a/tests/test_parsers_wifinetworks.py b/tests/test_parsers_wifinetworks.py index 268249b..20cc98b 100644 --- a/tests/test_parsers_wifinetworks.py +++ b/tests/test_parsers_wifinetworks.py @@ -1,31 +1,21 @@ -from parsers.wifinetworks import parsewifinetwork +from parsers.wifinetworks import parsewifinetwork, get_log_files from tests import SysdiagnoseTestCase -import os import unittest class TestParsersWifiNetworks(SysdiagnoseTestCase): - log_root_path = 'tests/testdata/iOS15/sysdiagnose_2023.05.24_13-29-15-0700_iPhone-OS_iPhone_19H349/' - log_files = [ - "WiFi/com.apple.wifi.known-networks.plist", - "WiFi/com.apple.wifi.plist", - "WiFi/LEGACY_com.apple.wifi-networks.plist", - "WiFi/com.apple.wifi-private-mac-networks.plist", - "WiFi/com.apple.wifi.recent-networks.json" - ] - def test_parsewifinetwork(self): - files = [os.path.join(self.log_root_path, log_file) for log_file in self.log_files] - result = parsewifinetwork(files) - self.assertGreater(len(result), 0) - for log_file in self.log_files: - end_of_filename = log_file.split('/')[-1] - self.assertTrue(end_of_filename in result) - if 'LEGACY' in end_of_filename: # skip empty file - continue - self.assertGreater(len(result[end_of_filename]), 0) + for log_root_path in self.log_root_paths: + files = get_log_files(log_root_path) + # self.assertTrue(len(files) > 0) + print(f'Parsing {files}') + result = parsewifinetwork(files) + self.assertGreater(len(result), 0) + for log_file in files: + end_of_filename = log_file.split('/')[-1] + self.assertTrue(end_of_filename in result) if __name__ == '__main__': diff --git a/tests/test_parsers_wifiscan.py b/tests/test_parsers_wifiscan.py index 5bbe088..6fd2688 100644 --- a/tests/test_parsers_wifiscan.py +++ b/tests/test_parsers_wifiscan.py @@ -5,8 +5,6 @@ class TestParsersWifiScan(SysdiagnoseTestCase): - # self.log_root_paths is defined in SysdiagnoseTestCase - def test_parsewifiscan(self): for log_root_path in self.log_root_paths: files = get_log_files(log_root_path)