From e3f89281339a67988220606d020ea07fc2afe469 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Thu, 12 Sep 2024 14:45:05 +0200 Subject: [PATCH] fix: [timesketch] fixes broken timesketch export --- README.md | 52 +++--- analysers/{timeliner.py => timesketch.py} | 155 ++++++++---------- parsers/logarchive.py | 7 +- ...eliner.py => test_analysers_timesketch.py} | 10 +- 4 files changed, 109 insertions(+), 115 deletions(-) rename analysers/{timeliner.py => timesketch.py} (70%) rename tests/{test_analysers_timeliner.py => test_analysers_timesketch.py} (54%) diff --git a/README.md b/README.md index 26bad98..4a1873f 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Create a virtual environment: sudo apt install graphviz ``` -On linux systems you may wish to install the unifiedlogs parser. See below for instructions how to do this. +On linux systems you may wish to install the [unifiedlogs](#unifiedlogs) parser. See below for instructions how to do this. # Quickstart @@ -38,13 +38,13 @@ Case ID: 1 Listing existing cases can be done easily: ``` -$ sysdiagnose list cases - Case ID Source file serial number ---------- ----------------------------------------------------------------------------------------------------------------- --------------- - 1 tests/testdata/iOS15/sysdiagnose_2023.05.24_13-29-15-0700_iPhone-OS_iPhone_19H349.tar.gz F4GT2K24HG7K +$ sysdiagnose cases +Case ID acquisition date Serial number Unique device ID iOS Version Tags +------------------- ------------------------- --------------- ---------------------------------------- ------------- ------ +public 2023-05-24T13:29:15-07:00 F4GT2K24HG7K e22f7f830e5dcc1287a1690a2622c2b12afaa33c ``` -The case folder is the current folder by default. +The `cases` folder is the current folder by default. You can change this using the environment variable `SYSDIAGNOSE_CASES_PATH`, for example. ``` $ export SYSDIAGNOSE_CASES_PATH='/path/to/folder' @@ -59,10 +59,10 @@ Run parsers: ``` $ sysdiagnose -c 1 parse ps -Execution success, output saved in: ./parsed_data/1/ps.json +Execution success, output saved in: cases/1/parsed_data/ps.json $ sysdiagnose -c 1 parse sys -Execution success, output saved in: ./parsed_data/1/sys.json +Execution success, output saved in: cases/1/parsed_data/sys.json ``` To run on all cases do not specify a case number or use `-c all`. @@ -117,7 +117,7 @@ apps Get list of Apps installed on the device demo_analyser Do something useful (DEMO) ps_everywhere List all processes we can find a bit everywhere. ps_matrix Makes a matrix comparing ps, psthread, taskinfo -timeliner Generate a Timesketch compatible timeline +timesketch Generate a Timesketch compatible timeline wifi_geolocation Generate GPS Exchange (GPX) of wifi geolocations wifi_geolocation_kml Generate KML file for wifi geolocations yarascan Scan the case folder using YARA rules ('./yara' or SYSDIAGNOSE_YARA_RULES_PATH) @@ -125,24 +125,21 @@ yarascan Scan the case folder using YARA rules ('./yara' or SYSDIAG Run analyser (make sure you run `parse all` before) ``` -$ sysdiagnose -c 1 analyse timeliner -Execution success, output saved in: ./parsed_data/1/timeliner.jsonl +$ sysdiagnose -c 1 analyse timesketch +Execution success, output saved in: cases/1/parsed_data/timesketch.jsonl ``` -Tested On: -- python 3.11 -- iOS13 (to be confirmed) -- iOS14 (to be confirmed) -- iOS15 -- iOS16 -- iOS17 -- iOS18 +# Using the output +Most of the parsers and analysers save their results in `jsonl` or `json` format. A few analysers use `txt` and more. +Exported data is stored in the `//parsed_data` folder. You can configure your ingestion tool to automatically monitor and all that data. -# Timesketch +The JSONL files are event based and (most often) structured with a a `timestamp` (unixtime) and `datetime` (isoformat) field. These can be used to build timelines. -You might want to visualise timelines which you can extract via sysdiagnose in [Timesketch](https://timesketch.org/guides/admin/install/). -Note that for a reasonable sysdiagnose log output, we recommend the following base requirements: +## Timesketch + +You might want to visualise timelines which you can extract via sysdiagnose in [Timesketch](https://timesketch.org/guides/admin/install/). Do note that timesketch expects timestamps in microseconds, that's why we made the `timesketch` analyser. +Note that for a reasonable sysdiagnose log output, we recommend the following base requirements: - Ubuntu 20.04 or higher - 128GB of RAM - 4-8 virtual CPUs @@ -171,6 +168,17 @@ sudo cp ../target/release/unifiedlog_parser_json /usr/local/bin/ See `unifiedlog_parser_json --help` for more instructions to use the tool, or use it directly through sysdiagnose. +# Supported iOS versions + +Tested On: +- python 3.11 +- iOS13 (to be confirmed) +- iOS14 (to be confirmed) +- iOS15 +- iOS16 +- iOS17 +- iOS18 + # Contributors diff --git a/analysers/timeliner.py b/analysers/timesketch.py similarity index 70% rename from analysers/timeliner.py rename to analysers/timesketch.py index a4632a8..b95f348 100644 --- a/analysers/timeliner.py +++ b/analysers/timesketch.py @@ -19,7 +19,7 @@ from utils.base import BaseAnalyserInterface -class TimelinerAnalyser(BaseAnalyserInterface): +class TimesketchAnalyser(BaseAnalyserInterface): description = 'Generate a Timesketch compatible timeline' format = 'jsonl' @@ -37,7 +37,7 @@ def __extract_ts_mobileactivation(self) -> Generator[dict, None, None]: data = p.get_result() for event in data: ts_event = { - 'message': 'Mobile Activation', + 'message': event['msg'], 'timestamp': event['timestamp'] * 1000000, 'datetime': event['datetime'], 'timestamp_desc': 'Mobile Activation Time' @@ -47,7 +47,7 @@ def __extract_ts_mobileactivation(self) -> Generator[dict, None, None]: except KeyError: # skip other type of event # FIXME what should we do? the log file (now) contains nice timestamps, do we want to extract less, but summarized, data? - continue + pass yield ts_event except Exception as e: print(f"ERROR while extracting timestamp from mobileactivation file. Reason: {str(e)}") @@ -56,13 +56,40 @@ def __extract_ts_powerlogs(self) -> Generator[dict, None, None]: try: p = PowerLogsParser(self.config, self.case_id) data = p.get_result() - # extract tables of interest - for entry in TimelinerAnalyser.__powerlogs__PLProcessMonitorAgent_EventPoint_ProcessExit(data): - yield entry - for entry in TimelinerAnalyser.__powerlogs__PLProcessMonitorAgent_EventBackward_ProcessExitHistogram(data): - yield entry - for entry in TimelinerAnalyser.__powerlogs__PLAccountingOperator_EventNone_Nodes(data): - yield entry + for event in data: + if event.get('db_table') == 'PLProcessMonitorAgent_EventPoint_ProcessExit': + extra_field = '' + if 'IsPermanent' in event: + extra_field = f"Is permanent: {event['IsPermanent']}" + ts_event = { + 'message': event['ProcessName'], + 'timestamp': event['timestamp'] * 1000000, + 'datetime': event['datetime'], + 'timestamp_desc': f"Process Exit with reason code: {event['ReasonCode']} reason namespace {event['ReasonNamespace']}", + 'extra_field_1': extra_field + } + yield ts_event + elif event.get('db_table') == 'PLProcessMonitorAgent_EventBackward_ProcessExitHistogram': + ts_event = { + 'message': event['ProcessName'], + 'timestamp': event['timestamp'] * 1000000, + 'datetime': event['datetime'], + 'timestamp_desc': f"Process Exit with reason code: {event['ReasonCode']} reason namespace {event['ReasonNamespace']}", + 'extra_field_1': f"Crash frequency: [0-5s]: {event['0s-5s']}, [5-10s]: {event['5s-10s']}, [10-60s]: {event['10s-60s']}, [60s+]: {event['60s+']}" + } + yield ts_event + elif event.get('db_table') == 'PLAccountingOperator_EventNone_Nodes': + ts_event = { + 'message': event['Name'], + 'timestamp': event['timestamp'] * 1000000, + 'datetime': event['datetime'], + 'timestamp_desc': 'PLAccountingOperator Event', + 'extra_field_1': f"Is permanent: {event['IsPermanent']}" + } + yield ts_event + else: + pass + except Exception as e: print(f"ERROR while extracting timestamp from powerlogs. Reason: {str(e)}") @@ -78,7 +105,7 @@ def __extract_ts_swcutil(self) -> Generator[dict, None, None]: 'message': service['Service'], 'timestamp': timestamp.timestamp() * 1000000, 'datetime': timestamp.isoformat(), - 'timestamp_desc': 'swcutil last checkeed', + 'timestamp_desc': 'swcutil last checked', 'extra_field_1': f"application: {service['App ID']}" } yield ts_event @@ -114,21 +141,19 @@ def __extract_ts_shutdownlogs(self) -> Generator[dict, None, None]: try: p = ShutdownLogsParser(self.config, self.case_id) data = p.get_result() - for ts, processes in data.items(): + for event in data: try: # create timeline entries - timestamp = datetime.strptime(ts, '%Y-%m-%d %H:%M:%S%z') - for p in processes: - ts_event = { - 'message': p['path'], - 'timestamp': timestamp.timestamp() * 1000000, - 'datetime': timestamp.isoformat(), - 'timestamp_desc': 'Entry in shutdown.log', - 'extra_field_1': f"pid: {p['pid']}" - } - yield ts_event + ts_event = { + 'message': event['path'], + 'timestamp': event['timestamp'] * 1000000, + 'datetime': event['datetime'], + 'timestamp_desc': 'Entry in shutdown.log', + 'extra_field_1': f"pid: {event['pid']}" + } + yield ts_event except Exception as e: - print(f"WARNING: shutdownlog entry not parsed: {ts}. Reason: {str(e)}") + print(f"WARNING: shutdownlog entry not parsed: {event}. Reason: {str(e)}") except Exception as e: print(f"ERROR while extracting timestamp from shutdownlog. Reason: {str(e)}") @@ -136,20 +161,19 @@ def __extract_ts_logarchive(self) -> Generator[dict, None, None]: try: p = LogarchiveParser(self.config, self.case_id) data = p.get_result() - for trace in data: + for event in data: try: # create timeline entry - timestamp = LogarchiveParser.convert_unifiedlog_time_to_datetime(trace['time']) ts_event = { - 'message': trace['message'], - 'timestamp': timestamp.timestamp() * 1000000, - 'datetime': trace['datetime'], - 'timestamp_desc': f"Entry in logarchive: {trace['event_type']}", - 'extra_field_1': f"subsystem: {trace['subsystem']}; process_uuid: {trace['process_uuid']}; process: {trace['process']}; library: {trace['library']}; library_uuid: {trace['library_uuid']}" + 'message': event['message'], + 'timestamp': event['timestamp'] * 1000000, + 'datetime': event['datetime'], + 'timestamp_desc': f"Entry in logarchive: {event['event_type']}", + 'extra_field_1': f"subsystem: {event['subsystem']}; process_uuid: {event['process_uuid']}; process: {event['process']}; library: {event['library']}; library_uuid: {event['library_uuid']}" } yield ts_event except KeyError as e: - print(f"WARNING: trace not parsed: {trace}. Error {e}") + print(f"WARNING: trace not parsed: {event}. Error {e}") except Exception as e: print(f"ERROR while extracting timestamp from logarchive. Reason: {str(e)}") @@ -248,18 +272,20 @@ def __extract_ts_crashlogs(self) -> Generator[dict, None, None]: p = CrashLogsParser(self.config, self.case_id) data = p.get_result() # process summary - for event in data.get('summary', []): - if event['datetime'] == '': - continue - timestamp = datetime.fromisoformat(event['datetime']) - ts_event = { - 'message': f"Application {event['app']} crashed.", - 'timestamp': timestamp.timestamp() * 1000000, - 'datetime': event['datetime'], - 'timestamp_desc': 'Application crash' - } - yield ts_event - # no need to also process the detailed crashes, as we already have the summary + for event in data: + try: + if event['datetime'] == '': + continue + ts_event = { + 'message': f"Application {event['app_name']} crashed.", + 'timestamp': event['timestamp'] * 1000000, + 'datetime': event['datetime'], + 'timestamp_desc': 'Application crash' + } + yield ts_event + # no need to also process the detailed crashes, as we already have the summary + except KeyError as e: + print(f"ERROR while extracting timestamp from crashlog for event {event}. Reason {str(e)}") except Exception as e: print(f"ERROR while extracting timestamp from crashlog. Reason {str(e)}") @@ -271,46 +297,3 @@ def execute(self): if func.startswith(f"_{self.__class__.__name__}__extract_ts_"): for event in getattr(self, func)(): # call the function yield event - - def __powerlogs__PLProcessMonitorAgent_EventPoint_ProcessExit(jdata): - proc_exit = jdata.get('PLProcessMonitorAgent_EventPoint_ProcessExit', []) - for proc in proc_exit: - timestamp = datetime.fromtimestamp(proc['timestamp'], tz=timezone.utc) - - extra_field = '' - if 'IsPermanent' in proc.keys(): - extra_field = f"Is permanent: {proc['IsPermanent']}" - ts_event = { - 'message': proc['ProcessName'], - 'timestamp': proc['timestamp'] * 1000000, - 'datetime': timestamp.isoformat(), - 'timestamp_desc': f"Process Exit with reason code: {proc['ReasonCode']} reason namespace {proc['ReasonNamespace']}", - 'extra_field_1': extra_field - } - yield ts_event - - def __powerlogs__PLProcessMonitorAgent_EventBackward_ProcessExitHistogram(jdata): - events = jdata.get('PLProcessMonitorAgent_EventBackward_ProcessExitHistogram', []) - for event in events: - timestamp = datetime.fromtimestamp(event['timestamp'], tz=timezone.utc) - ts_event = { - 'message': event['ProcessName'], - 'timestamp': event['timestamp'] * 1000000, - 'datetime': timestamp.isoformat(), - 'timestamp_desc': f"Process Exit with reason code: {event['ReasonCode']} reason namespace {event['ReasonNamespace']}", - 'extra_field_1': f"Crash frequency: [0-5s]: {event['0s-5s']}, [5-10s]: {event['5s-10s']}, [10-60s]: {event['10s-60s']}, [60s+]: {event['60s+']}" - } - yield ts_event - - def __powerlogs__PLAccountingOperator_EventNone_Nodes(jdata): - eventnone = jdata.get('PLAccountingOperator_EventNone_Nodes', []) - for event in eventnone: - timestamp = datetime.fromtimestamp(event['timestamp'], tz=timezone.utc) - ts_event = { - 'message': event['Name'], - 'timestamp': event['timestamp'] * 1000000, - 'datetime': timestamp.isoformat(), - 'timestamp_desc': 'PLAccountingOperator Event', - 'extra_field_1': f"Is permanent: {event['IsPermanent']}" - } - yield ts_event diff --git a/parsers/logarchive.py b/parsers/logarchive.py index 0053a11..d21e314 100644 --- a/parsers/logarchive.py +++ b/parsers/logarchive.py @@ -212,7 +212,9 @@ def convert_entry_to_unifiedlog_format(entry: dict) -> dict: ''' # already in the Mandiant unifiedlog format if 'event_type' in entry: - entry['datetime'] = LogarchiveParser.convert_unifiedlog_time_to_datetime(entry['time']).isoformat() + timestamp = LogarchiveParser.convert_unifiedlog_time_to_datetime(entry['time']) + entry['datetime'] = timestamp.isoformat() + entry['timestamp'] = timestamp.timestamp() return entry ''' jq '. |= keys' logarchive-native.json > native_keys.txt @@ -259,7 +261,8 @@ def convert_entry_to_unifiedlog_format(entry: dict) -> dict: new_entry[key] = value # convert time new_entry['datetime'] = new_entry['time'] - new_entry['time'] = LogarchiveParser.convert_native_time_to_unifiedlog_format(new_entry['time']) + new_entry['timestamp'] = datetime.fromisoformat(new_entry['time']).timestamp() + new_entry['time'] = new_entry['timestamp'] * 1000000000 return new_entry diff --git a/tests/test_analysers_timeliner.py b/tests/test_analysers_timesketch.py similarity index 54% rename from tests/test_analysers_timeliner.py rename to tests/test_analysers_timesketch.py index ceda167..9cf50ac 100644 --- a/tests/test_analysers_timeliner.py +++ b/tests/test_analysers_timesketch.py @@ -1,15 +1,15 @@ -from analysers.timeliner import TimelinerAnalyser +from analysers.timesketch import TimesketchAnalyser from tests import SysdiagnoseTestCase import unittest import os -class TestAnalysersTimeliner(SysdiagnoseTestCase): +class TestAnalysersTimesketch(SysdiagnoseTestCase): - def test_analyse_timeliner(self): + def test_analyse_timsketch(self): for case_id, case in self.sd.cases().items(): - print(f"Running Timeliner for {case_id}") - a = TimelinerAnalyser(self.sd.config, case_id=case_id) + print(f"Running Timesketch export for {case_id}") + a = TimesketchAnalyser(self.sd.config, case_id=case_id) a.save_result(force=True) self.assertTrue(os.path.isfile(a.output_file))