From 5a8663906aaebe4dd0994145081b27d25be9afe2 Mon Sep 17 00:00:00 2001 From: Jeremiah K <17190268+jeremiah-k@users.noreply.github.com> Date: Thu, 28 Nov 2024 20:40:34 -0600 Subject: [PATCH 1/9] Upgrade trunk, bump version Upgrade linters Ignore certain bandit linter subprocess warnings Move .bandit to .trunk/configs --- .gitignore | 2 +- .trunk/.gitignore | 2 +- .trunk/configs/.bandit | 2 ++ .trunk/trunk.yaml | 20 +++++++++++++------- app/downloader.py | 8 +++++--- app/menu_firmware.py | 3 ++- setup.cfg | 2 +- 7 files changed, 25 insertions(+), 14 deletions(-) create mode 100644 .trunk/configs/.bandit diff --git a/.gitignore b/.gitignore index 7cd094d..b7dc81d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ topic.txt fetchtastic.log .aider* app/__pycache__/ -fetchtastic.egg-info/ \ No newline at end of file +fetchtastic.egg-info/ diff --git a/.trunk/.gitignore b/.trunk/.gitignore index 072b680..15966d0 100644 --- a/.trunk/.gitignore +++ b/.trunk/.gitignore @@ -6,4 +6,4 @@ plugins user_trunk.yaml user.yaml -tmp \ No newline at end of file +tmp diff --git a/.trunk/configs/.bandit b/.trunk/configs/.bandit new file mode 100644 index 0000000..5f1a8be --- /dev/null +++ b/.trunk/configs/.bandit @@ -0,0 +1,2 @@ +[bandit] +skips: B404,B603,B607,B605,B311 diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index 855fca0..c8b2bb1 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -7,7 +7,7 @@ cli: plugins: sources: - id: trunk - ref: v1.6.4 + ref: v1.6.5 uri: https://github.com/trunk-io/plugins # Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes) runtimes: @@ -18,15 +18,21 @@ runtimes: lint: enabled: - actionlint@1.7.4 - - bandit@1.7.10 + - bandit@1.8.0 - black@24.10.0 - - checkov@3.2.291 + - checkov@3.2.322 - git-diff-check - isort@5.13.2 - - markdownlint@0.42.0 + - markdownlint@0.43.0 - osv-scanner@1.9.1 - - prettier@3.3.3 - - ruff@0.7.3 + - prettier@3.4.1 + - ruff@0.8.1 - taplo@0.9.3 - - trufflehog@3.83.6 + - trufflehog@3.84.1 - yamllint@1.35.1 +actions: + enabled: + - trunk-announce + - trunk-check-pre-push + - trunk-fmt-pre-commit + - trunk-upgrade-available diff --git a/app/downloader.py b/app/downloader.py index efb281b..efdf1a6 100644 --- a/app/downloader.py +++ b/app/downloader.py @@ -2,10 +2,10 @@ import json import os +import re import time import zipfile from datetime import datetime -import re import requests from requests.adapters import HTTPAdapter @@ -32,7 +32,9 @@ def main(): extract_patterns = config.get("EXTRACT_PATTERNS", []) exclude_patterns = config.get("EXCLUDE_PATTERNS", []) wifi_only = config.get("WIFI_ONLY", False) if setup_config.is_termux() else False - notify_on_download_only = config.get("NOTIFY_ON_DOWNLOAD_ONLY", False) # Added this line + notify_on_download_only = config.get( + "NOTIFY_ON_DOWNLOAD_ONLY", False + ) # Added this line selected_apk_patterns = config.get("SELECTED_APK_ASSETS", []) selected_firmware_patterns = config.get("SELECTED_FIRMWARE_ASSETS", []) @@ -180,7 +182,7 @@ def strip_version_numbers(filename): Uses the same regex as in menu_firmware.py to ensure consistency. """ # Regular expression matching version numbers and commit hashes - base_name = re.sub(r'([_-])\d+\.\d+\.\d+(?:\.[\da-f]+)?', r'\1', filename) + base_name = re.sub(r"([_-])\d+\.\d+\.\d+(?:\.[\da-f]+)?", r"\1", filename) return base_name # Cleanup function to keep only specific versions based on release tags diff --git a/app/menu_firmware.py b/app/menu_firmware.py index 6a72721..e6029bd 100644 --- a/app/menu_firmware.py +++ b/app/menu_firmware.py @@ -1,6 +1,7 @@ # app/menu_firmware.py import re + import requests from pick import pick @@ -32,7 +33,7 @@ def extract_base_name(filename): """ # Regular expression to match version numbers and commit hashes # Matches patterns like '-2.5.13.1a06f88' or '_2.5.13.1a06f88' - base_name = re.sub(r'([_-])\d+\.\d+\.\d+(?:\.[\da-f]+)?', r'\1', filename) + base_name = re.sub(r"([_-])\d+\.\d+\.\d+(?:\.[\da-f]+)?", r"\1", filename) return base_name diff --git a/setup.cfg b/setup.cfg index 4aef044..3aae056 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = fetchtastic -version = 0.2.3 +version = 0.2.4 author = Jeremiah K author_email = jeremiahk@gmx.com description = Meshtastic Firmware and APK Downloader From e30c23618691bc7811af5735fde68968dce2d355 Mon Sep 17 00:00:00 2001 From: Jeremiah K <17190268+jeremiah-k@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:01:36 -0600 Subject: [PATCH 2/9] Don't extract files unless necessary --- app/downloader.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/app/downloader.py b/app/downloader.py index efdf1a6..0852fd3 100644 --- a/app/downloader.py +++ b/app/downloader.py @@ -156,12 +156,18 @@ def extract_files(zip_path, extract_dir, patterns, exclude_patterns): stripped_base_name = strip_version_numbers(base_name) for pattern in patterns: if pattern in stripped_base_name: - # Extract and flatten directory structure - source = zip_ref.open(file_info) + # Check if the file already exists target_path = os.path.join(extract_dir, base_name) - with open(target_path, "wb") as target_file: - target_file.write(source.read()) - log_message(f"Extracted {base_name} to {extract_dir}") + if not os.path.exists(target_path): # Added check here + # Extract and flatten directory structure + source = zip_ref.open(file_info) + with open(target_path, "wb") as target_file: + target_file.write(source.read()) + log_message(f"Extracted {base_name} to {extract_dir}") + else: + log_message( + f"{base_name} already exists, skipping extraction." + ) matched_files.append(base_name) break # Stop checking patterns for this file if not matched_files: @@ -252,7 +258,10 @@ def check_and_download( zip_path = os.path.join(release_dir, file_name) if os.path.exists(zip_path): extraction_needed = check_extraction_needed( - zip_path, release_dir, extract_patterns + zip_path, + release_dir, + extract_patterns, + exclude_patterns, # Added exclude_patterns here ) if extraction_needed: log_message(f"Extracting files from {zip_path}...") @@ -318,11 +327,12 @@ def check_and_download( return downloaded_versions, new_versions_available - def check_extraction_needed(zip_path, extract_dir, patterns): + def check_extraction_needed(zip_path, extract_dir, patterns, exclude_patterns): """ Checks if extraction is needed based on the current extraction patterns. Returns True if any files matching the patterns are not already extracted. """ + files_to_extract = [] # Modified to collect files to extract with zipfile.ZipFile(zip_path, "r") as zip_ref: for file_info in zip_ref.infolist(): file_name = file_info.filename @@ -330,13 +340,20 @@ def check_extraction_needed(zip_path, extract_dir, patterns): # Skip directories if not base_name: continue + # Check if file matches exclude patterns + if any(exclude in base_name for exclude in exclude_patterns): + continue # Strip version numbers from the file name stripped_base_name = strip_version_numbers(base_name) for pattern in patterns: if pattern in stripped_base_name: - extracted_file_path = os.path.join(extract_dir, base_name) - if not os.path.exists(extracted_file_path): - return True # Extraction needed + files_to_extract.append(base_name) + break # Stop checking patterns for this file + # Now check if any of the files to extract are missing + for base_name in files_to_extract: + extracted_file_path = os.path.join(extract_dir, base_name) + if not os.path.exists(extracted_file_path): + return True # Extraction needed return False # All files already extracted start_time = time.time() From 7098ee8418260991e5e6f07d4113e9b7b44a4c3f Mon Sep 17 00:00:00 2001 From: Jeremiah K <17190268+jeremiah-k@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:15:04 -0600 Subject: [PATCH 3/9] Fix downloading missing versions --- app/downloader.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/app/downloader.py b/app/downloader.py index 0852fd3..dc8a179 100644 --- a/app/downloader.py +++ b/app/downloader.py @@ -170,10 +170,6 @@ def extract_files(zip_path, extract_dir, patterns, exclude_patterns): ) matched_files.append(base_name) break # Stop checking patterns for this file - if not matched_files: - log_message( - f"No files matched the extraction patterns in {zip_path}." - ) except zipfile.BadZipFile: log_message(f"Error: {zip_path} is a bad zip file and cannot be opened.") except Exception as e: @@ -247,7 +243,7 @@ def check_and_download( release_tag = release["tag_name"] release_dir = os.path.join(download_dir, release_tag) - if os.path.exists(release_dir) or release_tag == saved_release_tag: + if os.path.exists(release_dir): log_message(f"Processing existing version {release_tag}.") # Check if extraction is needed @@ -274,7 +270,6 @@ def check_and_download( else: # Files are already extracted pass - continue # Skip to the next release else: # Proceed to download this version os.makedirs(release_dir, exist_ok=True) From c2e223ba8dcae5eda50e7688734955f28368ee5b Mon Sep 17 00:00:00 2001 From: Jeremiah K <17190268+jeremiah-k@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:16:03 -0600 Subject: [PATCH 4/9] Update sample extraction patterns --- app/setup_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/setup_config.py b/app/setup_config.py index 7519955..fab2fc6 100644 --- a/app/setup_config.py +++ b/app/setup_config.py @@ -205,7 +205,7 @@ def run_setup(): print( "Enter the keywords to match for extraction from the firmware zip files, separated by spaces." ) - print("Example: rak4631- tbeam-2 t1000-e- tlora-v2-1-1_6- device-") + print("Example: rak4631- tbeam t1000-e- tlora-v2-1-1_6- device-") if config.get("EXTRACT_PATTERNS"): current_patterns = " ".join(config.get("EXTRACT_PATTERNS", [])) print(f"Current patterns: {current_patterns}") From 185cd4a0cbbc7f6a09d44c3fecb3257663f00646 Mon Sep 17 00:00:00 2001 From: Jeremiah K <17190268+jeremiah-k@users.noreply.github.com> Date: Tue, 3 Dec 2024 09:24:26 -0600 Subject: [PATCH 5/9] Set +x on extracted .sh files --- app/downloader.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/downloader.py b/app/downloader.py index dc8a179..f0f1af5 100644 --- a/app/downloader.py +++ b/app/downloader.py @@ -164,6 +164,12 @@ def extract_files(zip_path, extract_dir, patterns, exclude_patterns): with open(target_path, "wb") as target_file: target_file.write(source.read()) log_message(f"Extracted {base_name} to {extract_dir}") + # Set executable permissions if the file is a .sh script + if base_name.endswith(".sh"): + os.chmod(target_path, 0o755) + log_message( + f"Set executable permissions for {base_name}" + ) else: log_message( f"{base_name} already exists, skipping extraction." @@ -267,9 +273,7 @@ def check_and_download( extract_patterns, exclude_patterns, ) - else: - # Files are already extracted - pass + # No need to log that files are already extracted else: # Proceed to download this version os.makedirs(release_dir, exist_ok=True) From eb612a7a9ff3c08386c2242bd804d8ea2946baa2 Mon Sep 17 00:00:00 2001 From: Jeremiah K <17190268+jeremiah-k@users.noreply.github.com> Date: Tue, 3 Dec 2024 10:52:34 -0600 Subject: [PATCH 6/9] Refactor & only log when necessary --- .trunk/configs/.bandit | 2 +- app/downloader.py | 124 ++++++++++++++++++++++------------------- 2 files changed, 68 insertions(+), 58 deletions(-) diff --git a/.trunk/configs/.bandit b/.trunk/configs/.bandit index 5f1a8be..8aba310 100644 --- a/.trunk/configs/.bandit +++ b/.trunk/configs/.bandit @@ -1,2 +1,2 @@ [bandit] -skips: B404,B603,B607,B605,B311 +skips: B404,B603,B607,B605,B311,B103 diff --git a/app/downloader.py b/app/downloader.py index f0f1af5..660b50a 100644 --- a/app/downloader.py +++ b/app/downloader.py @@ -32,9 +32,7 @@ def main(): extract_patterns = config.get("EXTRACT_PATTERNS", []) exclude_patterns = config.get("EXCLUDE_PATTERNS", []) wifi_only = config.get("WIFI_ONLY", False) if setup_config.is_termux() else False - notify_on_download_only = config.get( - "NOTIFY_ON_DOWNLOAD_ONLY", False - ) # Added this line + notify_on_download_only = config.get("NOTIFY_ON_DOWNLOAD_ONLY", False) selected_apk_patterns = config.get("SELECTED_APK_ASSETS", []) selected_firmware_patterns = config.get("SELECTED_FIRMWARE_ASSETS", []) @@ -80,7 +78,8 @@ def send_ntfy_notification(message, title=None): except requests.exceptions.RequestException as e: log_message(f"Error sending notification to {ntfy_url}: {e}") else: - log_message("Notifications are not configured.") + # Don't log when notifications are not configured + pass # Function to get the latest releases and sort by date def get_latest_releases(url, scan_count=10): @@ -112,7 +111,8 @@ def download_file(url, download_path): file.write(chunk) log_message(f"Downloaded {download_path}") else: - log_message(f"{download_path} already exists, skipping download.") + # Don't log when the file already exists + pass except requests.exceptions.RequestException as e: log_message(f"Error downloading {url}: {e}") @@ -158,22 +158,20 @@ def extract_files(zip_path, extract_dir, patterns, exclude_patterns): if pattern in stripped_base_name: # Check if the file already exists target_path = os.path.join(extract_dir, base_name) - if not os.path.exists(target_path): # Added check here + if not os.path.exists(target_path): # Extract and flatten directory structure source = zip_ref.open(file_info) with open(target_path, "wb") as target_file: target_file.write(source.read()) log_message(f"Extracted {base_name} to {extract_dir}") - # Set executable permissions if the file is a .sh script - if base_name.endswith(".sh"): + # If the file is a .sh script, check permissions + if base_name.endswith(".sh"): + # Check if the file has executable permissions + if not os.access(target_path, os.X_OK): os.chmod(target_path, 0o755) log_message( f"Set executable permissions for {base_name}" ) - else: - log_message( - f"{base_name} already exists, skipping extraction." - ) matched_files.append(base_name) break # Stop checking patterns for this file except zipfile.BadZipFile: @@ -224,6 +222,7 @@ def check_and_download( ): downloaded_versions = [] new_versions_available = [] + actions_taken = False # Track if any actions were taken if not os.path.exists(download_dir): os.makedirs(download_dir) @@ -249,10 +248,35 @@ def check_and_download( release_tag = release["tag_name"] release_dir = os.path.join(download_dir, release_tag) - if os.path.exists(release_dir): - log_message(f"Processing existing version {release_tag}.") + # Create release directory if it doesn't exist + if not os.path.exists(release_dir): + os.makedirs(release_dir, exist_ok=True) - # Check if extraction is needed + assets_to_download = [] + for asset in release["assets"]: + file_name = asset["name"] + # Strip version numbers from the file name + stripped_file_name = strip_version_numbers(file_name) + # Matching logic + if selected_patterns: + if not any( + pattern in stripped_file_name for pattern in selected_patterns + ): + continue # Skip this asset + download_path = os.path.join(release_dir, file_name) + if not os.path.exists(download_path): + assets_to_download.append( + (asset["browser_download_url"], download_path) + ) + + if assets_to_download: + actions_taken = True + log_message(f"Downloading missing assets for version {release_tag}.") + for url, path in assets_to_download: + download_file(url, path) + downloaded_versions.append(release_tag) + + # Extraction logic if auto_extract and release_type == "Firmware": for asset in release["assets"]: file_name = asset["name"] @@ -263,7 +287,7 @@ def check_and_download( zip_path, release_dir, extract_patterns, - exclude_patterns, # Added exclude_patterns here + exclude_patterns, ) if extraction_needed: log_message(f"Extracting files from {zip_path}...") @@ -273,36 +297,12 @@ def check_and_download( extract_patterns, exclude_patterns, ) - # No need to log that files are already extracted else: - # Proceed to download this version - os.makedirs(release_dir, exist_ok=True) - log_message(f"Downloading new {release_type} version: {release_tag}") - for asset in release["assets"]: - file_name = asset["name"] - # Strip version numbers from the file name - stripped_file_name = strip_version_numbers(file_name) - # Matching logic - if selected_patterns: - if not any( - pattern in stripped_file_name - for pattern in selected_patterns - ): - continue # Skip this asset - download_path = os.path.join(release_dir, file_name) - download_file(asset["browser_download_url"], download_path) - if ( - auto_extract - and file_name.endswith(".zip") - and release_type == "Firmware" - ): - extract_files( - download_path, - release_dir, - extract_patterns, - exclude_patterns, - ) - downloaded_versions.append(release_tag) + # No action needed for this release + pass + + # Set permissions on .sh files if needed + set_permissions_on_sh_files(release_dir) # Only update latest_release_file if downloads occurred if downloaded_versions: @@ -315,6 +315,9 @@ def check_and_download( # Clean up old versions cleanup_old_versions(download_dir, release_tags_to_keep) + if not actions_taken: + log_message(f"All {release_type} assets are up to date.") + # Collect new versions available for release in releases_to_download: release_tag = release["tag_name"] @@ -326,12 +329,24 @@ def check_and_download( return downloaded_versions, new_versions_available + def set_permissions_on_sh_files(directory): + """ + Sets executable permissions on .sh files if they do not already have them. + """ + for root, dirs, files in os.walk(directory): + for file in files: + if file.endswith(".sh"): + file_path = os.path.join(root, file) + if not os.access(file_path, os.X_OK): + os.chmod(file_path, 0o755) + log_message(f"Set executable permissions for {file}") + def check_extraction_needed(zip_path, extract_dir, patterns, exclude_patterns): """ Checks if extraction is needed based on the current extraction patterns. Returns True if any files matching the patterns are not already extracted. """ - files_to_extract = [] # Modified to collect files to extract + files_to_extract = [] with zipfile.ZipFile(zip_path, "r") as zip_ref: for file_info in zip_ref.infolist(): file_name = file_info.filename @@ -398,9 +413,8 @@ def check_extraction_needed(zip_path, extract_dir, patterns, exclude_patterns): ) downloaded_firmwares.extend(fw_downloaded) new_firmware_versions.extend(fw_new_versions) - log_message( - f"Latest Firmware releases: {', '.join(release['tag_name'] for release in latest_firmware_releases[:versions_to_download])}" - ) + if fw_downloaded: + log_message(f"Downloaded Firmware versions: {', '.join(fw_downloaded)}") elif not selected_firmware_patterns: log_message("No firmware assets selected. Skipping firmware download.") @@ -420,9 +434,8 @@ def check_extraction_needed(zip_path, extract_dir, patterns, exclude_patterns): ) downloaded_apks.extend(apk_downloaded) new_apk_versions.extend(apk_new_versions) - log_message( - f"Latest Android APK releases: {', '.join(release['tag_name'] for release in latest_android_releases[:versions_to_download])}" - ) + if apk_downloaded: + log_message(f"Downloaded Android APK versions: {', '.join(apk_downloaded)}") elif not selected_apk_patterns: log_message("No APK assets selected. Skipping APK download.") @@ -466,12 +479,9 @@ def check_extraction_needed(zip_path, extract_dir, patterns, exclude_patterns): ) else: # No new downloads; everything is up to date - message = ( - f"No new downloads. All Firmware and Android APK versions are up to date.\n" - f"{datetime.now()}" - ) + message = f"All assets are up to date.\n" f"{datetime.now()}" log_message(message) - if not notify_on_download_only: # Added this condition + if not notify_on_download_only: send_ntfy_notification(message, title="Fetchtastic Up to Date") From 161b0ea688253ada8da0872c9c42fb15139f8b15 Mon Sep 17 00:00:00 2001 From: Jeremiah K <17190268+jeremiah-k@users.noreply.github.com> Date: Tue, 3 Dec 2024 11:01:22 -0600 Subject: [PATCH 7/9] dirs var intentionally unused _dirs --- app/downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/downloader.py b/app/downloader.py index 660b50a..b21542a 100644 --- a/app/downloader.py +++ b/app/downloader.py @@ -333,7 +333,7 @@ def set_permissions_on_sh_files(directory): """ Sets executable permissions on .sh files if they do not already have them. """ - for root, dirs, files in os.walk(directory): + for root, _dirs, files in os.walk(directory): for file in files: if file.endswith(".sh"): file_path = os.path.join(root, file) From d43e0ce42389bd1f8c8939fc119a6cad9aaa7ec9 Mon Sep 17 00:00:00 2001 From: Jeremiah K <17190268+jeremiah-k@users.noreply.github.com> Date: Tue, 3 Dec 2024 11:02:45 -0600 Subject: [PATCH 8/9] Remove unused variable --- app/downloader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/downloader.py b/app/downloader.py index b21542a..6b6ab6f 100644 --- a/app/downloader.py +++ b/app/downloader.py @@ -398,7 +398,6 @@ def check_extraction_needed(zip_path, extract_dir, patterns, exclude_patterns): latest_android_releases = [] if save_firmware and selected_firmware_patterns: - versions_to_download = firmware_versions_to_keep latest_firmware_releases = get_latest_releases( firmware_releases_url, releases_to_scan ) @@ -419,7 +418,6 @@ def check_extraction_needed(zip_path, extract_dir, patterns, exclude_patterns): log_message("No firmware assets selected. Skipping firmware download.") if save_apks and selected_apk_patterns: - versions_to_download = android_versions_to_keep latest_android_releases = get_latest_releases( android_releases_url, releases_to_scan ) From 646a91b49293fcfb86c59757b11e0f820eebf4fe Mon Sep 17 00:00:00 2001 From: Jeremiah K <17190268+jeremiah-k@users.noreply.github.com> Date: Tue, 3 Dec 2024 11:04:09 -0600 Subject: [PATCH 9/9] trunk upgrade --- .trunk/trunk.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index c8b2bb1..ce99594 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -20,7 +20,7 @@ lint: - actionlint@1.7.4 - bandit@1.8.0 - black@24.10.0 - - checkov@3.2.322 + - checkov@3.2.327 - git-diff-check - isort@5.13.2 - markdownlint@0.43.0 @@ -28,7 +28,7 @@ lint: - prettier@3.4.1 - ruff@0.8.1 - taplo@0.9.3 - - trufflehog@3.84.1 + - trufflehog@3.84.2 - yamllint@1.35.1 actions: enabled: