From c94690b72115408a43cbfac5d20ead0f30eaf813 Mon Sep 17 00:00:00 2001 From: Christophe Vandeplas Date: Fri, 30 Aug 2024 07:23:37 +0200 Subject: [PATCH] fix: [wifi_geolocation] fixes forgotten element in migration --- analysers/wifi_geolocation.py | 17 +--- analysers/yarascan.py | 163 +++++++++++++++++----------------- requirements.txt | 3 +- 3 files changed, 85 insertions(+), 98 deletions(-) diff --git a/analysers/wifi_geolocation.py b/analysers/wifi_geolocation.py index f9e2381..78bd972 100644 --- a/analysers/wifi_geolocation.py +++ b/analysers/wifi_geolocation.py @@ -3,12 +3,11 @@ # For Python3 # Author: Aaron Kaplan -import json import dateutil.parser -import os import gpxpy import gpxpy.gpx from utils.base import BaseAnalyserInterface +from parsers.wifi_known_networks import WifiKnownNetworksParser class WifiGeolocationAnalyser(BaseAnalyserInterface): @@ -25,19 +24,7 @@ def save_result(self, force: bool = False, indent=None): self.execute() def execute(self): - potential_source_files = ['wifinetworks/WiFi_com.apple.wifi.known-networks.plist.json', 'plists/WiFi_com.apple.wifi.known-networks.plist.json', 'wifi_known_networks.json'] - input_file_path = None - for fname in potential_source_files: - input_file_path = os.path.join(self.case_parsed_data_folder, fname) - if os.path.isfile(input_file_path): - break - if not input_file_path: - # TODO we could call the parser and generate the file for us...and then parse it... - raise FileNotFoundError(f"Could not find any of the potential source files: {potential_source_files}.") - - # we have a valid file_path and can generate the gpx file - with open(input_file_path, 'r') as f: - json_data = json.load(f) + json_data = WifiKnownNetworksParser(self.config, self.case_id).get_result() return WifiGeolocationAnalyser.generate_gpx_from_known_networks_json(json_data=json_data, output_file=self.output_file) def generate_gpx_from_known_networks_json(json_data: str, output_file: str): diff --git a/analysers/yarascan.py b/analysers/yarascan.py index 16f900e..aa2e7a1 100644 --- a/analysers/yarascan.py +++ b/analysers/yarascan.py @@ -38,7 +38,7 @@ def execute(self): if not os.path.isdir(self.yara_rules_path): raise FileNotFoundError(f"Could not find the YARA rules folder: {self.yara_rules_path}") - rule_files, errors = self.get_valid_yara_rule_files(self.yara_rules_path) + rule_files, errors = self.get_valid_yara_rule_files() if errors: results['errors'] = errors if len(rule_files) == 0: @@ -48,7 +48,7 @@ def execute(self): namespace = rule_file[len(self.yara_rules_path):].strip(os.path.sep) rule_filepaths[namespace] = rule_file - matches, errors = scan_directory( + matches, errors = YaraAnalyser.scan_directory( [ self.case_parsed_data_folder, self.case_data_folder @@ -67,7 +67,7 @@ def execute(self): return results - def get_valid_yara_rule_files(self, rules_path: str) -> tuple[list, list]: + def get_valid_yara_rule_files(self) -> tuple[list, list]: rule_files_to_test = glob.glob(os.path.join(self.yara_rules_path, '**', '*.yar'), recursive=True) rule_files_validated = [] errors = [] @@ -90,90 +90,89 @@ def get_valid_yara_rule_files(self, rules_path: str) -> tuple[list, list]: return rule_files_validated, errors + def scan_directory(directories: list, rule_filepaths: dict, ignore_files: list, ignore_folders: list) -> tuple[list, list]: + results_lock = threading.Lock() + matches = {} + errors = [] -def scan_directory(directories: list, rule_filepaths: dict, ignore_files: list, ignore_folders: list) -> tuple[list, list]: - results_lock = threading.Lock() - matches = {} - errors = [] - - # multi-threaded file scanning, really speeds up if multiple large files are present + # multi-threaded file scanning, really speeds up if multiple large files are present - # build and fill the queue - file_queue = queue.Queue() - for directory in directories: - for root, _, files in os.walk(directory): - stop = False - for ignore_folder in ignore_folders: - if root.startswith(ignore_folder): - stop = True - print(f"Skipping folder: {root}") - continue - if stop: - continue - for file in files: - file_full_path = os.path.join(root, file) + # build and fill the queue + file_queue = queue.Queue() + for directory in directories: + for root, _, files in os.walk(directory): stop = False - for ignore_file in ignore_files: - if file_full_path.startswith(ignore_file): + for ignore_folder in ignore_folders: + if root.startswith(ignore_folder): stop = True - print(f"Skipping file: {file_full_path}") + print(f"Skipping folder: {root}") continue if stop: continue - file_queue.put(file_full_path) - - # define our consumer that will run in the threads - def consumer(): - # compile rules only once ... and ignore file specific externals. Massive speedup - rules = yara.compile(filepaths=rule_filepaths, externals=externals) - - while True: - print(f"Consumer thread seeing {file_queue.qsize()} files in queue, and taking one") - file_path = file_queue.get() - if file_path is None: - print("Consumer thread exiting") - break - - print(f"Scanning file: {file_path}") - # set the externals for this file - massive slowdown - # externals_local = externals.copy() - # externals_local['filename'] = file - # externals_local['filepath'] = file_path[len(directory) + 1:] # exclude the case root directory that installation specific - # externals_local['extension'] = os.path.splitext(file)[1] - # rules = yara.compile(filepaths=rule_filepaths, externals=externals_local) - try: - m = rules.match(file_path) - if m: - key = file_path[len(directory) + 1:] + for file in files: + file_full_path = os.path.join(root, file) + stop = False + for ignore_file in ignore_files: + if file_full_path.startswith(ignore_file): + stop = True + print(f"Skipping file: {file_full_path}") + continue + if stop: + continue + file_queue.put(file_full_path) + + # define our consumer that will run in the threads + def consumer(): + # compile rules only once ... and ignore file specific externals. Massive speedup + rules = yara.compile(filepaths=rule_filepaths, externals=externals) + + while True: + print(f"Consumer thread seeing {file_queue.qsize()} files in queue, and taking one") + file_path = file_queue.get() + if file_path is None: + print("Consumer thread exiting") + break + + print(f"Scanning file: {file_path}") + # set the externals for this file - massive slowdown + # externals_local = externals.copy() + # externals_local['filename'] = file + # externals_local['filepath'] = file_path[len(directory) + 1:] # exclude the case root directory that installation specific + # externals_local['extension'] = os.path.splitext(file)[1] + # rules = yara.compile(filepaths=rule_filepaths, externals=externals_local) + try: + m = rules.match(file_path) + if m: + key = file_path[len(directory) + 1:] + with results_lock: + matches[key] = {} + for match in m: + matches[key][match.rule] = { + 'tags': match.tags, + 'meta': match.meta, + 'strings': [str(s) for s in match.strings], + 'rule_file': match.namespace + } + except yara.Error as e: with results_lock: - matches[key] = {} - for match in m: - matches[key][match.rule] = { - 'tags': match.tags, - 'meta': match.meta, - 'strings': [str(s) for s in match.strings], - 'rule_file': match.namespace - } - except yara.Error as e: - with results_lock: - errors.append(f"Error matching file {file_path}: {e}") - file_queue.task_done() # signal that the file has been processed - - max_threads = os.cpu_count() * 2 or 4 # default to 4 if we can't determine the number of CPUs - # Create and start consumer threads - consumer_threads = [] - for _ in range(max_threads): - t = threading.Thread(target=consumer) - t.start() - consumer_threads.append(t) - - # Wait for the queue to be empty - file_queue.join() - - # Stop the consumer threads - for _ in range(max_threads): - file_queue.put(None) - for t in consumer_threads: - t.join() - - return matches, errors + errors.append(f"Error matching file {file_path}: {e}") + file_queue.task_done() # signal that the file has been processed + + max_threads = os.cpu_count() * 2 or 4 # default to 4 if we can't determine the number of CPUs + # Create and start consumer threads + consumer_threads = [] + for _ in range(max_threads): + t = threading.Thread(target=consumer) + t.start() + consumer_threads.append(t) + + # Wait for the queue to be empty + file_queue.join() + + # Stop the consumer threads + for _ in range(max_threads): + file_queue.put(None) + for t in consumer_threads: + t.join() + + return matches, errors diff --git a/requirements.txt b/requirements.txt index 1d0d350..c58cf19 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ python-dateutil==2.9.0.post0 gpxpy==1.6.2 pandas==2.2.2 nska-deserialize==1.4.0 -yara-python==4.5.1 \ No newline at end of file +yara-python==4.5.1 +# pycrashreport==1.2.4 \ No newline at end of file