diff --git a/.github/workflows/pycodestyle.yml b/.github/workflows/pycodestyle.yml index 323314c..5158d21 100644 --- a/.github/workflows/pycodestyle.yml +++ b/.github/workflows/pycodestyle.yml @@ -28,4 +28,4 @@ jobs: - name: Analysing the code with pycodestyle run: | - pycodestyle --exclude=venv --ignore=errors=E221,E225,E251,E501,E266,E302 --max-line-length=128 $(git ls-files '*.py') + pycodestyle --exclude=venv --ignore=errors=E302,E221,E225,E251,E501,E266,E302 --max-line-length=128 $(git ls-files '*.py') diff --git a/.gitignore b/.gitignore index 43d7d76..df22e57 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ data/* cases.json parsed_data/* *.gpx +db.json # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/README.md b/README.md index f675629..825bcdf 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Tested On: - iOS13 - iOS14 - iOS16 -- iOS17.0 (ongoing) +- iOS17 # Timesketch @@ -75,6 +75,15 @@ Note that for a reasonable sysdiagnose log output, we recommend the following ba - Minimum 64 GB of HDD space just for timesketch data (add some more GBs for the OS and OS upgrades, etc.) - SSDs (NVMEs) for the data. +# Contributors + +- Dario BORREGUERO RINCON (European Commission - EC DIGIT Cybersecurity Operation Centre) +- David DURVAUX (European Commission - EC DIGIT Cybersecurity Operation Centre) +- Aaron KAPLAN (European Commission - EC DIGIT Cybersecurity Operation Centre) +- Emilien LE JAMTEL (CERT-EU) +- Benoît ROUSSILLE (European Parliament) + + # License This project is released under the European Public Licence https://commission.europa.eu/content/european-union-public-licence_en diff --git a/analyzers/apps.py b/analyzers/apps.py index 3ba78f6..fe8acf7 100644 --- a/analyzers/apps.py +++ b/analyzers/apps.py @@ -15,11 +15,11 @@ -v --version Show version. """ import os -from optparse import OptionParser +# from optparse import OptionParser import json import ijson from docopt import docopt -import glob +# import glob version_string = "sysdiagnose-demo-analyser.py v2023-04-28 Version 0.1" @@ -32,6 +32,7 @@ # --------------------------------------------------------------------------- # + def apps_analysis(jsondir, filename): """ Go through all json files in the folder and generate the markdown file @@ -94,6 +95,8 @@ def apps_analysis(jsondir, filename): """ Main function """ + + def main(): """ Main function, to be called when used as CLI tool diff --git a/analyzers/sysdiagnose-timeliner.py b/analyzers/sysdiagnose-timeliner.py index bdd42f8..8e569a3 100644 --- a/analyzers/sysdiagnose-timeliner.py +++ b/analyzers/sysdiagnose-timeliner.py @@ -29,6 +29,7 @@ "sysdiagnose-mobileactivation.json": "__extract_ts_mobileactivation", "sysdiagnose-powerlogs.json": "__extract_ts_powerlogs", "sysdiagnose-swcutil.json": "__extract_ts_swcutil", + "sysdiagnose-shutdownlogs.json": "__extract_ts_shutdownlogs", "sysdiagnose-logarchive.json": "__extract_ts_logarchive", "sysdiagnose-wifisecurity.json": "__extract_ts_wifisecurity", "sysdiagnose_wifi_known_networks.json": "__extract_ts_wifi_known_networks", @@ -49,7 +50,7 @@ def __extract_ts_mobileactivation(filename): timestamp = datetime.strptime(event["timestamp"], "%Y-%m-%d %H:%M:%S") ts_event = { "message": "Mobile Activation", - "timestamp": int(timestamp.timestamp()*1000000), + "timestamp": int(timestamp.timestamp() * 1000000), "datetime": timestamp.strftime("%Y-%m-%dT%H:%M:%S+00:00"), "timestamp_desc": "Mobile Activation Time", "extra_field_1": "Build Version: %s" % event["build_version"] @@ -84,12 +85,16 @@ def __extract_ts_powerlogs__PLProcessMonitorAgent_EventPoint_ProcessExit(jdata): proc_exit = jdata["PLProcessMonitorAgent_EventPoint_ProcessExit"] for proc in proc_exit: timestamp = datetime.fromtimestamp(int(proc["timestamp"])) + + extra_field = "" + if "IsPermanent" in proc.keys(): + extra_field = "Is permanent: %d" % proc["IsPermanent"] ts_event = { "message": proc["ProcessName"], - "timestamp": int(timestamp.timestamp()*1000000), + "timestamp": int(timestamp.timestamp() * 1000000), "datetime": timestamp.strftime("%Y-%m-%dT%H:%M:%S+00:00"), - "timestamp_desc": "Process Exit with reason code: %d reason namespace" % (proc["ReasonCode"], proc["ReasonNamespace"]), - "extra_field_1": "Is permanent: %d" % proc["IsPermanent"], + "timestamp_desc": "Process Exit with reason code: %d reason namespace %d" % (proc["ReasonCode"], proc["ReasonNamespace"]), + "extra_field_1": extra_field } timeline.append(ts_event) return @@ -101,9 +106,9 @@ def __extract_ts_powerlogs__PLProcessMonitorAgent_EventBackward_ProcessExitHisto timestamp = datetime.fromtimestamp(int(event["timestamp"])) ts_event = { "message": event["ProcessName"], - "timestamp": int(timestamp.timestamp()*1000000), + "timestamp": int(timestamp.timestamp() * 1000000), "datetime": timestamp.strftime("%Y-%m-%dT%H:%M:%S+00:00"), - "timestamp_desc": "Process Exit with reason code: %d reason namespace" % (event["ReasonCode"], event["ReasonNamespace"]), + "timestamp_desc": "Process Exit with reason code: %d reason namespace %d" % (event["ReasonCode"], event["ReasonNamespace"]), "extra_field_1": "Crash frequency: [0-5s]: %d, [5-10s]: %d, [10-60s]: %d, [60s+]: %d" % (event["0s-5s"], event["5s-10s"], event["10s-60s"], event["60s+"]) } timeline.append(ts_event) @@ -116,7 +121,7 @@ def __extract_ts_powerlogs__PLAccountingOperator_EventNone_Nodes(jdata): timestamp = datetime.fromtimestamp(int(event["timestamp"])) ts_event = { "message": event["Name"], - "timestamp": int(timestamp.timestamp()*1000000), + "timestamp": int(timestamp.timestamp() * 1000000), "datetime": timestamp.strftime("%Y-%m-%dT%H:%M:%S+00:00"), "timestamp_desc": "PLAccountingOperator Event", "extra_field_1": "Is permanent: %d" % event["IsPermanent"] @@ -139,25 +144,23 @@ def __extract_ts_swcutil(filename): "Next Check": "2023-02-28 22:06:35 +0000" }, """ - try: - with open(filename, 'r') as fd: - data = json.load(fd) - if "db" in data.keys(): - for service in data["db"]: + with open(filename, 'r') as fd: + data = json.load(fd) + if "db" in data.keys(): + for service in data["db"]: + try: timestamp = datetime.strptime(service["Last Checked"], "%Y-%m-%d %H:%M:%S %z") ts_event = { "message": service["Service"], - "timestamp": int(timestamp.timestamp()*1000000), + "timestamp": int(timestamp.timestamp() * 1000000), "datetime": timestamp.strftime("%Y-%m-%dT%H:%M:%S+00:00"), "timestamp_desc": "swcutil last checkeed", "extra_field_1": "application: %s" % service["App ID"] } timeline.append(ts_event) - return True - except Exception as e: - print(f"ERROR while extracting timestamp from {filename}. Reason: {str(e)}") - return False - return False + except Exception as e: + print(f"ERROR {filename} while extracting timestamp from {(service['Service'])} - {(service['App ID'])}. Record not inserted.") + return True def __extract_ts_accessibility_tcc(filename): @@ -182,21 +185,14 @@ def __extract_ts_accessibility_tcc(filename): data = json.load(fd) if "access" in data.keys(): for access in data["access"]: - # convert to hashtable - service = {} - for line in access: - keys = line.keys() - for key in keys: - service[key] = line[key] - # create timeline entry - timestamp = datetime.fromtimestamp(int(service["last_modified"])) + timestamp = datetime.fromtimestamp(int(access["last_modified"])) ts_event = { - "message": service["service"], - "timestamp": int(timestamp.timestamp()*1000000), + "message": access["service"], + "timestamp": int(timestamp.timestamp() * 1000000), "datetime": timestamp.strftime("%Y-%m-%dT%H:%M:%S+00:00"), "timestamp_desc": "Accessibility TC Last Modified", - "extra_field_1": "client: %s" % service["client"] + "extra_field_1": "client: %s" % access["client"] } timeline.append(ts_event) return True @@ -205,6 +201,32 @@ def __extract_ts_accessibility_tcc(filename): return False return False +def __extract_ts_shutdownlogs(filename): + try: + with open(filename, 'r') as fd: + data = json.load(fd) + for ts in data["data"].keys(): + try: + # create timeline entries + timestamp = datetime.strptime(ts, "%Y-%m-%d %H:%M:%S+00:00") + processes = data["data"][ts] + for p in processes: + ts_event = { + "message": p["path"], + "timestamp": int(timestamp.timestamp() * 1000000), + "datetime": timestamp.strftime("%Y-%m-%dT%H:%M:%S+00:00"), + "timestamp_desc": "Entry in shutdown.log", + "extra_field_1": "pid: %s" % p["pid"] + } + timeline.append(ts_event) + except Exception as e: + print(f"WARNING: entry not parsed: {ts}") + return True + except Exception as e: + print(f"ERROR while extracting timestamp from {filename}. Reason: {str(e)}") + return False + return False + def __extract_ts_logarchive(filename): r""" @@ -250,7 +272,7 @@ def __extract_ts_logarchive(filename): timestamp = datetime.strptime(trace["timestamp"], "%Y-%m-%d %H:%M:%S.%f%z") ts_event = { "message": trace["eventMessage"], - "timestamp": int(timestamp.timestamp()*1000000), + "timestamp": int(timestamp.timestamp() * 1000000), "datetime": timestamp.strftime("%Y-%m-%dT%H:%M:%S+00:00"), "timestamp_desc": "Entry in logarchive: %s" % trace["eventType"], "extra_field_1": "subsystem: %s; processImageUUID: %s; processImagePath: %s" % (trace["subsystem"], trace["processImageUUID"], trace["processImagePath"]) @@ -291,7 +313,7 @@ def __extract_ts_wifisecurity(filename): # Event 1: creation ts_event = { "message": wifi["acct"], - "timestamp": int(ctimestamp.timestamp()*1000000), + "timestamp": int(ctimestamp.timestamp() * 1000000), "datetime": ctimestamp.strftime("%Y-%m-%dT%H:%M:%S+00:00"), "timestamp_desc": "SSID added to known secured WIFI list", "extra_field_1": wifi["accc"] @@ -301,7 +323,7 @@ def __extract_ts_wifisecurity(filename): # Event 2: modification ts_event = { "message": wifi["acct"], - "timestamp": int(mtimestamp.timestamp()*1000000), + "timestamp": int(mtimestamp.timestamp() * 1000000), "datetime": mtimestamp.strftime("%Y-%m-%dT%H:%M:%S+00:00"), "timestamp_desc": "SSID modified into the secured WIFI list", "extra_field_1": wifi["accc"] @@ -315,14 +337,12 @@ def __extract_ts_wifisecurity(filename): def __extract_ts_wifi_known_networks(filename): - try: - with open(filename, 'r') as fd: - data = json.load(fd) - for wifi in data.keys(): - ssid = data[wifi]["SSID"] + with open(filename, 'r') as fd: + data = json.load(fd) + for wifi in data.keys(): + ssid = data[wifi]["SSID"] + try: added = datetime.strptime(data[wifi]["AddedAt"], "%Y-%m-%d %H:%M:%S.%f") - updated = datetime.strptime(data[wifi]["UpdatedAt"], "%Y-%m-%d %H:%M:%S.%f") - modified_password = datetime.strptime(data[wifi]["__OSSpecific__"]["WiFiNetworkPasswordModificationDate"], "%Y-%m-%d %H:%M:%S.%f") # WIFI added ts_event = { @@ -333,8 +353,12 @@ def __extract_ts_wifi_known_networks(filename): "extra_field_1": "Add reason: %s" % data[wifi]["AddReason"] } timeline.append(ts_event) + except Exception as e: + print(f"ERROR {filename} while extracting timestamp from {ssid}. Reason: {str(e)}. Record not inserted.") # WIFI modified + try: + updated = datetime.strptime(data[wifi]["UpdatedAt"], "%Y-%m-%d %H:%M:%S.%f") ts_event = { "message": "WIFI %s added" % updated, "timestamp": updated.timestamp(), @@ -343,8 +367,12 @@ def __extract_ts_wifi_known_networks(filename): "extra_field_1": "Add reason: %s" % data[wifi]["AddReason"] } timeline.append(ts_event) + except Exception as e: + print(f"ERROR {filename} while extracting timestamp from {ssid}. Reason: {str(e)}. Record not inserted.") # Password for wifi modified + try: + modified_password = datetime.strptime(data[wifi]["__OSSpecific__"]["WiFiNetworkPasswordModificationDate"], "%Y-%m-%d %H:%M:%S.%f") ts_event = { "message": "Password for WIFI %s modified" % ssid, "timestamp": modified_password.timestamp(), @@ -353,12 +381,10 @@ def __extract_ts_wifi_known_networks(filename): "extra_field_1": "AP mode: %s" % data[wifi]["__OSSpecific__"]["AP_MODE"] } timeline.append(ts_event) + except Exception as e: + print(f"ERROR {filename} while extracting timestamp from {ssid}. Reason: {str(e)}. Record not inserted.") - return True - except Exception as e: - print(f"ERROR while extracting timestamp from {filename}. Reason: {str(e)}") - return False - return False + return True def parse_json(jsondir): diff --git a/analyzers/sysdiagnose-wifi-gelocation-kml.py b/analyzers/sysdiagnose-wifi-gelocation-kml.py new file mode 100644 index 0000000..157e9fa --- /dev/null +++ b/analyzers/sysdiagnose-wifi-gelocation-kml.py @@ -0,0 +1,130 @@ +#! /usr/bin/env python3 + +# For Python3 +# Author: Aaron Kaplan + +import sys +import json +# import dateutil.parser +from optparse import OptionParser + +import xml.etree.ElementTree as ET + + +sys.path.append('..') # noqa: E402 +# from sysdiagnose import config # noqa: E402 + + +version_string = "sysdiagnose-wifi-geolocation-kml.py v2023-08-09 Version 0.2" + +# ----- definition for analyse.py script -----# +# ----- DO NOT DELETE ----# + +analyser_description = "Generate KML file for wifi geolocations" +analyser_call = "generate_kml" +analyser_format = "json" + + +def generate_kml(jsonfile: str, outfile: str = "wifi-geolocations.kml"): + """ + Modify the function to generate a KML file from Wi-Fi geolocation data and include a tour between the locations. + Reads as input and extracts all known Wi-Fi networks and their locations. + """ + try: + with open(jsonfile, 'r') as fp: + json_data = json.load(fp) + except Exception as e: + print(f"Error while parsing inputfile JSON. Reason: {str(e)}") + sys.exit(-1) + + json_entry = json_data.get('com.apple.wifi.known-networks.plist') + if not json_entry: + print("Could not find the 'com.apple.wifi.known-networks.plist' section. Bailing out.", file=sys.stderr) + sys.exit(-2) + json_data = json_entry + + # Create new KML root + kml = ET.Element('kml', xmlns='http://www.opengis.net/kml/2.2') + document = ET.SubElement(kml, 'Document') + + # Add tour elements + tour = ET.SubElement(document, 'gx:Tour') + ET.SubElement(tour, 'name').text = 'WiFi Tour' + playlist = ET.SubElement(tour, 'gx:Playlist') + + for network_name, network_data in json_data.items(): + ssid = network_name + lat = network_data.get('Latitude') + lon = network_data.get('Longitude') + + if lat and lon: + placemark = ET.SubElement(document, 'Placemark') + ET.SubElement(placemark, 'name').text = ssid + point = ET.SubElement(placemark, 'Point') + ET.SubElement(point, 'coordinates').text = f"{lon},{lat},0" + + # Add to tour playlist + flyto = ET.SubElement(playlist, 'gx:FlyTo') + ET.SubElement(flyto, 'gx:duration').text = '5.0' # Duration of each flyto + ET.SubElement(flyto, 'gx:flyToMode').text = 'smooth' + camera = ET.SubElement(flyto, 'Camera') + ET.SubElement(camera, 'longitude').text = str(lon) + ET.SubElement(camera, 'latitude').text = str(lat) + ET.SubElement(camera, 'altitude').text = '500' # Camera altitude + ET.SubElement(camera, 'heading').text = '0' + ET.SubElement(camera, 'tilt').text = '45' + ET.SubElement(camera, 'roll').text = '0' + + # Convert the ElementTree to a string and save it to a file + tree = ET.ElementTree(kml) + tree.write(outfile) + + # Example usage: + # generate_kml_with_tour('input_json_file.json', 'output_kml_file.kml') + + return + + +# --------------------------------------------------------------------------- # +""" + Main function +""" + + +def main(): + + print(f"Running {version_string}\n") + + usage = "\n%prog -d JSON directory\n" + + parser = OptionParser(usage=usage) + parser.add_option("-i", dest="inputfile", + action="store", type="string", + help="JSON file from parsers") + parser.add_option("-o", dest="outputfile", + action="store", type="string", + help="KML file to save output") + (options, args) = parser.parse_args() + + # no arguments given by user, print help and exit + if len(sys.argv) == 1: + parser.print_help() + sys.exit(-1) + + if not options.inputfile and not options.outputfile: + parser.error("Need to specify inputfile and outputfile") + else: + generate_kml(options.inputfile, outfile=options.outputfile) + + +# --------------------------------------------------------------------------- # + +""" + Call main function +""" +if __name__ == "__main__": + + # Create an instance of the Analysis class (called "base") and run main + main() + +# That's all folks ;) diff --git a/analyzers/sysdiagnose-wifi-gelocation.py b/analyzers/sysdiagnose-wifi-gelocation.py index dc8d812..61910ef 100644 --- a/analyzers/sysdiagnose-wifi-gelocation.py +++ b/analyzers/sysdiagnose-wifi-gelocation.py @@ -19,7 +19,7 @@ version_string = "sysdiagnose-demo-analyser.py v2023-04-28 Version 0.1" # ----- definition for analyse.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# analyser_description = "Generate GPS Exchange (GPX) of wifi geolocations" analyser_call = "generate_gpx" diff --git a/initialyze.py b/initialyze.py index 3deb36b..269aeec 100644 --- a/initialyze.py +++ b/initialyze.py @@ -160,6 +160,11 @@ def init(sysdiagnose_file, force=False): except: # noqa: E722 pass + try: + new_case_json["shutdownlog"] = new_folder +glob.glob('./*/system_logs.logarchive/Extra/shutdown.log')[0][1:] + except: # noqa: E722 + pass + try: new_case_json["taskinfo"] = new_folder +glob.glob('./*/taskinfo.txt')[0][1:] except: # noqa: E722 diff --git a/parsers/sysdiagnose-itunesstore.py b/parsers/sysdiagnose-itunesstore.py index f09bf37..c27fbf1 100644 --- a/parsers/sysdiagnose-itunesstore.py +++ b/parsers/sysdiagnose-itunesstore.py @@ -12,7 +12,7 @@ version_string = "sysdiagnose-itunesstore.py v2020-20-19 Version 1.0" # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing iTunes store logs" parser_input = "itunesstore" diff --git a/parsers/sysdiagnose-logarchive.py b/parsers/sysdiagnose-logarchive.py index a8151f2..56e5ec3 100644 --- a/parsers/sysdiagnose-logarchive.py +++ b/parsers/sysdiagnose-logarchive.py @@ -16,7 +16,7 @@ version_string = "sysdiagnose-logarchive.py v2020-02-07 Version 1.0" # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing system_logs.logarchive folder" parser_input = "logarchive_folder" diff --git a/parsers/sysdiagnose-mobileactivation.py b/parsers/sysdiagnose-mobileactivation.py index b647238..b24ef1d 100644 --- a/parsers/sysdiagnose-mobileactivation.py +++ b/parsers/sysdiagnose-mobileactivation.py @@ -27,7 +27,7 @@ # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing mobileactivation logs file" parser_input = "mobile_activation" # list of log files diff --git a/parsers/sysdiagnose-mobileinstallation.py b/parsers/sysdiagnose-mobileinstallation.py index 0d60634..1e5f745 100644 --- a/parsers/sysdiagnose-mobileinstallation.py +++ b/parsers/sysdiagnose-mobileinstallation.py @@ -27,7 +27,7 @@ # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing mobile_installation logs file" parser_input = "mobile_installation" # list of log files diff --git a/parsers/sysdiagnose-networkextension.py b/parsers/sysdiagnose-networkextension.py index 4da1e43..50c81fa 100644 --- a/parsers/sysdiagnose-networkextension.py +++ b/parsers/sysdiagnose-networkextension.py @@ -16,17 +16,17 @@ -v --version Show version. """ -import sys -from optparse import OptionParser -import plistlib +# import sys +# from optparse import OptionParser +# import plistlib import biplist -import json +# import json from docopt import docopt from tabulate import tabulate # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing networkextension plist file" parser_input = "networkextension" @@ -45,7 +45,7 @@ def parseplist(file): output = {'objects': []} for object in objects: - if type(object) == str: + if isinstance(object, str): output['objects'].append(object) return output diff --git a/parsers/sysdiagnose-networkextensioncache.py b/parsers/sysdiagnose-networkextensioncache.py index 37d0c0e..4f0ab56 100644 --- a/parsers/sysdiagnose-networkextensioncache.py +++ b/parsers/sysdiagnose-networkextensioncache.py @@ -26,7 +26,7 @@ # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing networkextensioncache plist file" parser_input = "networkextensioncache" diff --git a/parsers/sysdiagnose-olddsc.py b/parsers/sysdiagnose-olddsc.py index 83fc462..53ae61d 100644 --- a/parsers/sysdiagnose-olddsc.py +++ b/parsers/sysdiagnose-olddsc.py @@ -14,7 +14,7 @@ version_string = "sysdiagnose-olddsc.py v2020-02-26 Version 1.0" # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing olddsc files" parser_input = "olddsc" diff --git a/parsers/sysdiagnose-ps.py b/parsers/sysdiagnose-ps.py index ed70a4f..3ebe346 100644 --- a/parsers/sysdiagnose-ps.py +++ b/parsers/sysdiagnose-ps.py @@ -17,7 +17,7 @@ version_string = "sysdiagnose-ps.py v2023-03-10 Version 1.1" # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing ps.txt file" parser_input = "ps" @@ -62,8 +62,10 @@ def parse_ps(filename, ios_version=16): "STAT": patterns[12], "STARTED": patterns[13], "TIME": patterns[14], - "COMMAND": patterns[15]} + "COMMAND": "".join(patterns[15:])} else: + # Note: bellow - attempt to create a regex but feel it will more lead to errors. Instead lets merge all parts of the commands (patterns[17:]) + # regex = r"(?P\w+)\s+(?P\d+)\s+(?\d+|\-)\s+(?\d+)\s+(?\d+)\s+(?\d+)\s+(?\d+\.\d+)\s+(?\d+\.\d+)\s+(?\d+)\s+(?\d+)\s+(?\d+)\s+(?\-)" processes[int(patterns[3])] = {"USER": patterns[0], "UID": patterns[1], "PRSNA": patterns[2], @@ -81,7 +83,7 @@ def parse_ps(filename, ios_version=16): "STAT": patterns[14], "STARTED": patterns[15], "TIME": patterns[16], - "COMMAND": patterns[17]} + "COMMAND": "".join(patterns[17:])} fd.close() except Exception as e: diff --git a/parsers/sysdiagnose-psthread.py b/parsers/sysdiagnose-psthread.py index 79c226e..2dbc59e 100644 --- a/parsers/sysdiagnose-psthread.py +++ b/parsers/sysdiagnose-psthread.py @@ -17,7 +17,7 @@ version_string = "sysdiagnose-ps.py Version 1.0" # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing ps_thread.txt file" parser_input = "ps_thread" diff --git a/parsers/sysdiagnose-shutdownlogs.py b/parsers/sysdiagnose-shutdownlogs.py new file mode 100644 index 0000000..3e5bb76 --- /dev/null +++ b/parsers/sysdiagnose-shutdownlogs.py @@ -0,0 +1,110 @@ +#! /usr/bin/env python3 + +# For Python3 +# Demo blank parsers +# Author: david@autopsit.org + +import os +import sys +import json +from optparse import OptionParser +import time +import struct +import datetime +import re + +version_string = "sysdiagnose-shutdownlog.py v2024-01-11 Version 1.0" + +# ----- definition for parsing.py script -----# + +parser_description = "Parsing shutdown.log file" +parser_input = "shutdownlog" +parser_call = "parse_shutdownlog" + +# --------------------------------------------# + +CLIENTS_ARE_STILL_HERE_LINE = "these clients are still here" +REMAINING_CLIENT_PID_LINE = "remaining client pid" +SIGTERM_LINE = "SIGTERM" + +# --------------------------------------------------------------------------- # + + +def parse_shutdownlog(filepath, ios_version=16): + """ + This is the function that will be called + """ + # read log file content + log_lines = "" + with open(filepath, "r") as f: + log_lines = f.readlines() + + json_object = {} + parsed_data = {} + index = 0 + # go through log file + while index < len(log_lines): + # look for begining of shutdown sequence + if CLIENTS_ARE_STILL_HERE_LINE in log_lines[index]: + running_processes = [] + while not(SIGTERM_LINE in log_lines[index]): + if (REMAINING_CLIENT_PID_LINE in log_lines[index]): + result = re.search(r".*: (\b\d+) \((.*)\).*", log_lines[index]) + pid = result.groups()[0] + binary_path = result.groups()[1] + process = pid + ":" + binary_path + if not(process in running_processes): + running_processes.append(process) + index += 1 + # compute timestamp from SIGTERM line + result = re.search(r".*\[(\d+)\].*", log_lines[index]) + timestamp = result.groups()[0] + time = str(datetime.datetime.fromtimestamp(int(timestamp), datetime.UTC)) + # add entries + parsed_data[time] = [] + for p in running_processes: + parsed_data[time].append({"pid": p.split(":")[0], "path": p.split(":")[1]}) + index += 1 + + json_object["data"] = parsed_data + + return json_object + + +# --------------------------------------------------------------------------- # + +def main(): + """ + Main function, to be called when used as CLI tool + """ + + print(f"Running {version_string}\n") + + usage = "\n%prog -i inputfile\n" + + parser = OptionParser(usage=usage) + parser.add_option("-i", dest="inputfile", + action="store", type="string", + help="path to the shutdown.log file") + (options, args) = parser.parse_args() + + # no arguments given by user, print help and exit + if len(sys.argv) == 1: + parser.print_help() + sys.exit(-1) + + # Call the demo function when called directly from CLI + print(parse_shutdownlog(options.inputfile)) + +# --------------------------------------------------------------------------- # + + +""" + Call main function +""" +if __name__ == "__main__": + + # Create an instance of the Analysis class (called "base") and run main + main() + +# That's all folk ;) diff --git a/parsers/sysdiagnose-swcutil.py b/parsers/sysdiagnose-swcutil.py index 6ca189c..3edbc91 100644 --- a/parsers/sysdiagnose-swcutil.py +++ b/parsers/sysdiagnose-swcutil.py @@ -25,7 +25,7 @@ # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing swcutil_show file" parser_input = "swcutil_show" diff --git a/parsers/sysdiagnose-sys.py b/parsers/sysdiagnose-sys.py index 04471d3..89f30cb 100644 --- a/parsers/sysdiagnose-sys.py +++ b/parsers/sysdiagnose-sys.py @@ -13,7 +13,7 @@ version_string = "sysdiagnose-sys.py v2019-05-10 Version 2.0" # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing SystemVersion plist file" parser_input = "systemversion" diff --git a/parsers/sysdiagnose-taskinfo.py b/parsers/sysdiagnose-taskinfo.py index e22bbcc..48396ff 100644 --- a/parsers/sysdiagnose-taskinfo.py +++ b/parsers/sysdiagnose-taskinfo.py @@ -15,7 +15,7 @@ version_string = "sysdiagnose-taskinfo.py v2020-02-07 Version 1.0" # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing taskinfo txt file" parser_input = "taskinfo" diff --git a/parsers/sysdiagnose-uuid2path.py b/parsers/sysdiagnose-uuid2path.py index 646a651..3259314 100644 --- a/parsers/sysdiagnose-uuid2path.py +++ b/parsers/sysdiagnose-uuid2path.py @@ -16,7 +16,7 @@ version_string = "sysdiagnose-uuid2path.py v2020-02-07 Version 2.0" # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing UUIDToBinaryLocations plist file" parser_input = "UUIDToBinaryLocations" diff --git a/parsers/sysdiagnose_wifi_known_networks.py b/parsers/sysdiagnose_wifi_known_networks.py index bd126b1..0ec636c 100644 --- a/parsers/sysdiagnose_wifi_known_networks.py +++ b/parsers/sysdiagnose_wifi_known_networks.py @@ -27,7 +27,7 @@ def default(self, obj): version_string = "sysdiagnose_wifi_known_networks.py v2023-05-19 Version 1.0" # ----- definition for parsing.py script -----# -# ----- DO NET DELETE ----# +# ----- DO NOT DELETE ----# parser_description = "Parsing Known Wifi Networks plist file" parser_input = "wifi_data" diff --git a/parsing.py b/parsing.py index 0c93df3..31baa1e 100644 --- a/parsing.py +++ b/parsing.py @@ -116,7 +116,7 @@ def parse(parser, case_id): spec.loader.exec_module(module) # building command - if type(case[module.parser_input]) == str: + if isinstance(case[module.parser_input], str): command = 'module.' + module.parser_call + '(\'' + case[module.parser_input] + '\')' else: command = 'module.' + module.parser_call + '(' + str(case[module.parser_input]) + ')'