new: [parsers] move file filter to parsers #65

parsers/spindumpnosymbols.py

@@ -16,14 +16,13 @@
   -v --version     Show version.
 """
 
-import sys
-from optparse import OptionParser
-import plistlib
-import json
 from docopt import docopt
-from tabulate import tabulate
+import glob
+import json
+import os
 import re
-import pprint
+import sys
+
 
 # ----- definition for parsing.py script -----#
 # -----         DO NOT DELETE             ----#
@@ -35,6 +34,17 @@
 # --------------------------------------------#
 
 
+def get_log_files(log_root_path: str) -> list:
+    log_files_globs = [
+        'spindump-nosymbols.txt'
+    ]
+    log_files = []
+    for log_files_glob in log_files_globs:
+        log_files.extend(glob.glob(os.path.join(log_root_path, log_files_glob)))
+
+    return log_files
+
+
 def parsespindumpNS(file):
     with open(file, 'r') as f_in:
         # init section
@@ -167,7 +177,7 @@ def parse_thread(data):
     output["loaded"] = []
 
     for line in data[1:]:
-        loaded={}
+        loaded = {}
         if "+" in line:
             loaded["library"] = line.split("(", 1)[1].split("+", 1)[0].strip()
             loaded["int"] = line.split("(", 1)[1].split("+", 1)[1].split(")", 1)[0].strip()
@@ -179,7 +189,7 @@ def parse_thread(data):
 
 
 def parse_images(data):
-    images=[]
+    images = []
     for line in data:
         image = {}
         if line.strip() is not None:
@@ -203,12 +213,12 @@ def main():
 
     arguments = docopt(__doc__, version='parser for networkextension.plist v0.1')
 
-    ### test
+    # ## test
     if arguments['-i']:
         # Output is good enough, just print
         print(json.dumps(parsespindumpNS(arguments['<file>']), indent=4))
         sys.exit()
-    ### test
+    # ## test
 
     return 0
 

parsers/swcutil.py

@@ -16,13 +16,10 @@
   -v --version     Show version.
 """
 
-import sys
-from optparse import OptionParser
-import plistlib
-import json
 from docopt import docopt
-from tabulate import tabulate
-
+import glob
+import os
+import sys
 
 # ----- definition for parsing.py script -----#
 # -----         DO NOT DELETE             ----#
@@ -34,6 +31,17 @@
 # --------------------------------------------#
 
 
+def get_log_files(log_root_path: str) -> list:
+    log_files_globs = [
+        'swcutil_show.txt'
+    ]
+    log_files = []
+    for log_files_glob in log_files_globs:
+        log_files.extend(glob.glob(os.path.join(log_root_path, log_files_glob)))
+
+    return log_files
+
+
 def parseswcutil(file):
     with open(file, 'r') as f_in:
         # init section
@@ -115,15 +123,15 @@ def main():
 
     arguments = docopt(__doc__, version='parser for networkextension.plist v0.1')
 
-    ### test
+    # ## test
     if arguments['-i']:
         # Output is good enough, just print
         with open(arguments['<file>'], 'r') as f_in:
             for line in f_in:
                 print(line.strip())
         # parseswcutil(arguments['<file>'])
         sys.exit()
-    ### test
+    # ## test
 
     return 0
 

parsers/sys.py

@@ -9,6 +9,8 @@
 import sys
 from optparse import OptionParser
 import plistlib
+import os
+import glob
 
 version_string = "sysdiagnose-sys.py v2019-05-10 Version 2.0"
 
@@ -22,6 +24,17 @@
 # --------------------------------------------#
 
 
+def get_log_files(log_root_path: str) -> list:
+    log_files_globs = [
+        'logs/SystemVersion/SystemVersion.plist'
+    ]
+    log_files = []
+    for log_files_glob in log_files_globs:
+        log_files.extend(glob.glob(os.path.join(log_root_path, log_files_glob)))
+
+    return log_files
+
+
 # --------------------------------------------------------------------------- #
 # XXX FIXME: this is obviously a very generic function and could be generalized and moved to misc/
 

parsers/taskinfo.py

@@ -9,7 +9,8 @@
 #
 import re
 import sys
-import json
+import glob
+import os
 from optparse import OptionParser
 
 version_string = "sysdiagnose-taskinfo.py v2020-02-07 Version 1.0"
@@ -24,6 +25,17 @@
 # --------------------------------------------#
 
 
+def get_log_files(log_root_path: str) -> list:
+    log_files_globs = [
+        'taskinfo.txt'
+    ]
+    log_files = []
+    for log_files_glob in log_files_globs:
+        log_files.extend(glob.glob(os.path.join(log_root_path, log_files_glob)))
+
+    return log_files
+
+
 # --------------------------------------------------------------------------- #
 def get_num_tasks(filename, ios_version=13):
     """
@@ -264,7 +276,7 @@ def get_tasks(filename, ios_version=13):
                 continue
         fd.close()
     except Exception as e:
-        print(f"Could not open {filename}")
+        print(f"Could not open {filename} for reading. Reason: {str(e)}")
 
     return {"numb_tasks": numb_tasks, "tasks": tasks}
 

parsers/wifi_known_networks.py

@@ -13,7 +13,7 @@
 import biplist
 from biplist import Uid, Data
 from datetime import datetime
-
+import glob
 import config
 
 
@@ -36,6 +36,17 @@ def default(self, obj):
 # --------------------------------------------#
 
 
+def get_log_files(log_root_path: str) -> list:
+    log_files_globs = [
+        'WiFi/com.apple.wifi.known-networks.plist'
+    ]
+    log_files = []
+    for log_files_glob in log_files_globs:
+        log_files.extend(glob.glob(os.path.join(log_root_path, log_files_glob)))
+
+    return log_files
+
+
 # --------------------------------------------------------------------------- #
 # XXX #26 FIXME: this is obviously a very generic function and could be generalized and moved to misc/
 

parsers/wifinetworks.py

@@ -21,6 +21,7 @@
 import glob
 # sys.path.append('..')   # noqa: E402
 import misc        # noqa: E402
+import os
 
 # ----- definition for parsing.py script -----#
 
@@ -31,6 +32,18 @@
 # --------------------------------------------------------------------------- #
 
 
+def get_log_files(log_root_path: str) -> list:
+    log_files_globs = [
+        'WiFi/*.plist',
+        'WiFi/com.apple.wifi.recent-networks.json'
+    ]
+    log_files = []
+    for log_files_glob in log_files_globs:
+        log_files.extend(glob.glob(os.path.join(log_root_path, log_files_glob)))
+
+    return log_files
+
+
 def parsewifinetwork(wifi_data: list):
     output = {}
     for data in wifi_data:

tests/__init__.py

@@ -11,4 +11,4 @@ def setUp(self):
         # - testdata-private/iOSxx/sysdiagnose_YYYY.MM.DD_HH-MM-SS-SSSS_...
         # this allows testing locally with more data, while keeping online tests coherent
 
-        self.log_root_paths = [name for name in glob.glob('tests/testdata*/*/*') if os.path.isdir(name)]
+        self.log_root_paths = [name for name in glob.glob('tests/testdata*/**/sysdiagnose_*', recursive=True) if os.path.isdir(name)]

tests/test_parsers_spindumpnosymbols.py

@@ -1,18 +1,20 @@
-from parsers.spindumpnosymbols import parsespindumpNS
+from parsers.spindumpnosymbols import parsespindumpNS, get_log_files
 from tests import SysdiagnoseTestCase
 import unittest
 
 
 class TestParsersSpindumpnosymbols(SysdiagnoseTestCase):
 
-    log_path = "tests/testdata/iOS15/sysdiagnose_2023.05.24_13-29-15-0700_iPhone-OS_iPhone_19H349/spindump-nosymbols.txt"
-
     def test_parsespindumpNS(self):
-        result = parsespindumpNS(self.log_path)
-        self.assertGreater(len(result), 0)
-        self.assertTrue('iPhone OS' in result['OS Version'])
-        self.assertTrue('rootdev' in result['Boot args'])
-        self.assertGreater(len(result['processes']), 0)
+        for log_root_path in self.log_root_paths:
+            files = get_log_files(log_root_path)
+            self.assertTrue(len(files) > 0)
+            for file in files:
+                print(f'Parsing {file}')
+                result = parsespindumpNS(file)
+                self.assertGreater(len(result), 0)
+                self.assertTrue('OS Version' in result)
+                self.assertGreater(len(result['processes']), 0)
 
 
 if __name__ == '__main__':

tests/test_parsers_swcutil.py

@@ -1,24 +1,23 @@
-from parsers.swcutil import parseswcutil
+from parsers.swcutil import parseswcutil, get_log_files
 from tests import SysdiagnoseTestCase
 import unittest
 
 
 class TestParsersSwcutil(SysdiagnoseTestCase):
-
-    log_path = "tests/testdata/iOS15/sysdiagnose_2023.05.24_13-29-15-0700_iPhone-OS_iPhone_19H349/swcutil_show.txt"
-
-    db_keys = ['Service', 'App ID', 'App Version', 'App PI', 'Domain', 'Patterns', 'User Approval', 'Site/Fmwk Approval', 'Flags']
-
     def test_parseswcutil(self):
-        result = parseswcutil(self.log_path)
-        self.assertGreater(len(result), 0)
-        self.assertTrue('headers' in result)
-        self.assertTrue('network' in result)
-        self.assertTrue('headers' in result)
-
-        self.assertGreater(len(result['memory']), 0)
-        self.assertGreater(len(result['db']), 0)
-        self.assertEqual(list(result['db'][0].keys()), self.db_keys)
+        for log_root_path in self.log_root_paths:
+            files = get_log_files(log_root_path)
+            self.assertTrue(len(files) > 0)
+            for file in files:
+                print(f'Parsing {file}')
+                result = parseswcutil(file)
+                self.assertGreater(len(result), 0)
+                self.assertTrue('headers' in result)
+                self.assertTrue('network' in result)
+                self.assertTrue('headers' in result)
+
+                self.assertGreater(len(result['memory']), 0)
+                self.assertGreater(len(result['db']), 0)
 
 
 if __name__ == '__main__':

tests/test_parsers_sys.py

@@ -1,21 +1,23 @@
-from parsers.sys import getProductInfo
+from parsers.sys import getProductInfo, get_log_files
 from tests import SysdiagnoseTestCase
 import unittest
 
 
 class TestParsersSys(SysdiagnoseTestCase):
-
-    log_path = "tests/testdata/iOS15/sysdiagnose_2023.05.24_13-29-15-0700_iPhone-OS_iPhone_19H349/logs/SystemVersion/SystemVersion.plist"
-
     productinfo_keys = ['ProductName', 'ProductBuildVersion', 'ProductVersion', 'BuildID', 'SystemImageID']
 
     def test_getProductInfo(self):
-        result = getProductInfo(self.log_path)
-        self.assertGreater(len(result), 0)
-
-        self.assertTrue(result.keys() | self.productinfo_keys == result.keys())  # check if the result contains at least the following keys
-        self.assertTrue('iPhone OS' in result['ProductName'])
-        self.assertTrue(result['BuildID'])
+        for log_root_path in self.log_root_paths:
+            files = get_log_files(log_root_path)
+            self.assertTrue(len(files) > 0)
+            for file in files:
+                print(f'Parsing {file}')
+                result = getProductInfo(file)
+                self.assertGreater(len(result), 0)
+
+                self.assertTrue(result.keys() | self.productinfo_keys == result.keys())  # check if the result contains at least the following keys
+                self.assertTrue('iPhone OS' in result['ProductName'])
+                self.assertTrue(result['BuildID'])
 
 
 if __name__ == '__main__':

tests/test_parsers_taskinfo.py

@@ -1,18 +1,20 @@
-from parsers.taskinfo import get_tasks
+from parsers.taskinfo import get_tasks, get_log_files
 from tests import SysdiagnoseTestCase
 import unittest
 
 
 class TestParsersTaskinfo(SysdiagnoseTestCase):
 
-    log_path = "tests/testdata/iOS15/sysdiagnose_2023.05.24_13-29-15-0700_iPhone-OS_iPhone_19H349/taskinfo.txt"
-
     def test_get_tasks(self):
-        result = get_tasks(self.log_path)
-        self.assertGreater(len(result), 0)
-
-        self.assertEqual(result['numb_tasks'], 244)
-        self.assertGreater(len(result['tasks']), 0)
+        for log_root_path in self.log_root_paths:
+            files = get_log_files(log_root_path)
+            self.assertTrue(len(files) > 0)
+            for file in files:
+                print(f'Parsing {file}')
+                result = get_tasks(file)
+                self.assertGreater(len(result), 0)
+                self.assertGreater(result['numb_tasks'], 0)
+                self.assertGreater(len(result['tasks']), 0)
 
 
 if __name__ == '__main__':