From 9064f78799eee653cf0a2067ddbc975a7a6c6ee7 Mon Sep 17 00:00:00 2001 From: ya0211 Date: Mon, 12 Jun 2023 18:50:54 +0800 Subject: [PATCH 1/6] Fix method add_track when cover is True --- sync/track/LocalTracks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/track/LocalTracks.py b/sync/track/LocalTracks.py index 58a85fc..f241b1a 100644 --- a/sync/track/LocalTracks.py +++ b/sync/track/LocalTracks.py @@ -73,8 +73,8 @@ def add_track(cls, track, modules_folder, cover=True): track.write(json_file) elif cover: old = TrackJson.load(json_file) - track.added = old.added - track.write(json_file) + old.update(track) + old.write(json_file) @classmethod def del_track(cls, module_id, modules_folder): From eb53f712e3991c9674c3aa005bcd5eeef97142f7 Mon Sep 17 00:00:00 2001 From: ya0211 Date: Mon, 12 Jun 2023 19:03:27 +0800 Subject: [PATCH 2/6] Fix method _from_zip_common when needing to remove tmp files --- sync/core/Pull.py | 13 ++++++++----- sync/core/Pull.pyi | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/sync/core/Pull.py b/sync/core/Pull.py index 7227541..fcbdd36 100644 --- a/sync/core/Pull.py +++ b/sync/core/Pull.py @@ -79,6 +79,12 @@ def _get_changelog_common(self, module_id, changelog): def _from_zip_common(self, module_id, zip_file, changelog_file, *, delete_tmp=True): module_folder = self.modules_folder.joinpath(module_id) + def remove_file(): + if delete_tmp: + os.remove(zip_file) + if delete_tmp and changelog_file is not None: + os.remove(changelog_file) + zip_file_size = os.path.getsize(zip_file) / (1024 ** 2) if zip_file_size > self._max_size: module_folder.joinpath(LocalTracks.TAG_DISABLE).touch() @@ -95,11 +101,9 @@ def get_online_module(): result = get_online_module() if result.is_failure: - if delete_tmp: - os.remove(zip_file) - msg = Log.get_msg(result.error) self._log.e(f"_from_zip_common: [{module_id}] -> {msg}") + remove_file() return None else: online_module: OnlineModule = result.value @@ -110,8 +114,7 @@ def get_online_module(): if not target_zip_file.exists() and len(target_files) == 0: self._copy_file(zip_file, target_zip_file, delete_tmp) else: - if delete_tmp: - os.remove(zip_file) + remove_file() return None target_changelog_file = module_folder.joinpath(online_module.changelog_filename) diff --git a/sync/core/Pull.pyi b/sync/core/Pull.pyi index 4674bb7..1deb13f 100644 --- a/sync/core/Pull.pyi +++ b/sync/core/Pull.pyi @@ -25,7 +25,7 @@ class Pull: def _get_file_url(self, module_id: str, file: Path) -> str: ... def _get_changelog_common(self, module_id: str, changelog: str) -> Optional[Path]: ... def _from_zip_common( - self, module_id: str, zip_file: Path, changelog: Optional[Path], *, delete_tmp: bool = ... + self, module_id: str, zip_file: Path, changelog_file: Optional[Path], *, delete_tmp: bool = ... ) -> Optional[OnlineModule]: ... def from_json(self, track: TrackJson, *, local: bool) -> Tuple[Optional[OnlineModule], float]: ... def from_url(self, track: TrackJson) -> Tuple[Optional[OnlineModule], float]: ... From c9d0981432bf0b9552fe2c243e705ea4f2acc01e Mon Sep 17 00:00:00 2001 From: ya0211 Date: Mon, 12 Jun 2023 19:31:13 +0800 Subject: [PATCH 3/6] Add checks for changelog content --- sync/core/Pull.py | 47 ++++++++++++++++++++++------------------- sync/core/Pull.pyi | 1 + sync/utils/HttpUtils.py | 4 ++-- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/sync/core/Pull.py b/sync/core/Pull.py index fcbdd36..ff49f59 100644 --- a/sync/core/Pull.py +++ b/sync/core/Pull.py @@ -36,6 +36,14 @@ def _copy_file(old, new, delete_old=True): def _safe_download(url, out): return HttpUtils.download(url, out) + def _check_changelog(self, module_id, file): + text = file.read_text() + if HttpUtils.is_html(text): + self._log.w(f"_check_changelog: [{module_id}] -> unsupported changelog type [the content is html text]") + return False + else: + return True + def _get_file_url(self, module_id, file): module_folder = self.modules_folder.joinpath(module_id) url = f"{self._config.repo_url}{self.modules_folder.name}/{module_id}/{file.name}" @@ -49,30 +57,25 @@ def _get_changelog_common(self, module_id, changelog): if not isinstance(changelog, str) or changelog == "": return None - unsupported = False - changelog_file = None if changelog.startswith("http"): - if changelog.endswith("md"): - changelog_file = self.modules_folder.joinpath(module_id, f"{module_id}.md") - result = self._safe_download(changelog, changelog_file) - if result.is_failure: - msg = Log.get_msg(result.error) - self._log.e(f"_get_changelog_common: [{module_id}] -> {msg}") - changelog_file = None - else: - unsupported = True + changelog_file = self.modules_folder.joinpath(module_id, f"{module_id}.md") + result = self._safe_download(changelog, changelog_file) + if result.is_failure: + msg = Log.get_msg(result.error) + self._log.e(f"_get_changelog_common: [{module_id}] -> {msg}") + changelog_file = None else: - if changelog.endswith("md") or changelog.endswith("log"): - changelog_file = self._local_folder.joinpath(changelog) - if not changelog_file.exists(): - msg = f"_get_changelog_common: [{module_id}] -> {changelog} is not in {self._local_folder}" - self._log.i(msg) - changelog_file = None - else: - unsupported = True - - if unsupported: - self._log.w(f"_get_changelog_common: [{module_id}] -> unsupported changelog type [{changelog}]") + changelog_file = self._local_folder.joinpath(changelog) + if not changelog_file.exists(): + msg = f"_get_changelog_common: [{module_id}] -> {changelog} is not in {self._local_folder}" + self._log.i(msg) + changelog_file = None + + if changelog_file is not None: + is_target_type = self._check_changelog(module_id, changelog_file) + if not is_target_type: + os.remove(changelog_file) + changelog_file = None return changelog_file diff --git a/sync/core/Pull.pyi b/sync/core/Pull.pyi index 1deb13f..e960ed0 100644 --- a/sync/core/Pull.pyi +++ b/sync/core/Pull.pyi @@ -22,6 +22,7 @@ class Pull: @staticmethod @Result.catching() def _safe_download(url: str, out: Path) -> Result: ... + def _check_changelog(self, module_id: str, file: Path) -> bool: ... def _get_file_url(self, module_id: str, file: Path) -> str: ... def _get_changelog_common(self, module_id: str, changelog: str) -> Optional[Path]: ... def _from_zip_common( diff --git a/sync/utils/HttpUtils.py b/sync/utils/HttpUtils.py index f6d794f..487e786 100644 --- a/sync/utils/HttpUtils.py +++ b/sync/utils/HttpUtils.py @@ -31,7 +31,7 @@ def load_json(cls, url: str) -> Union[list, AttrDict]: return obj @classmethod - def _is_html(cls, text): + def is_html(cls, text): pattern = r'|||' return re.search(pattern, text, re.IGNORECASE) is not None @@ -52,7 +52,7 @@ def download(cls, url: str, out: Path) -> float: else: os.remove(out) - if cls._is_html(response.text): + if cls.is_html(response.text): msg = "404 not found" else: msg = response.text From f3b002d6a3338b1ae2f74b49c6cdc96ccf17c1e0 Mon Sep 17 00:00:00 2001 From: ya0211 Date: Tue, 13 Jun 2023 10:23:04 +0800 Subject: [PATCH 4/6] Optimize imports --- sync/cli/Main.py | 2 +- sync/cli/Parameters.py | 2 +- sync/core/Pull.py | 4 ++-- sync/core/Pull.pyi | 2 +- sync/modifier/Command.py | 3 ++- sync/utils/GitUtils.py | 2 +- sync/utils/Log.py | 8 ++++---- 7 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sync/cli/Main.py b/sync/cli/Main.py index 290abfc..c081042 100644 --- a/sync/cli/Main.py +++ b/sync/cli/Main.py @@ -1,8 +1,8 @@ import json import os import sys -from pathlib import Path from datetime import datetime +from pathlib import Path from .Parameters import Parameters from .SafeArgs import SafeArgs diff --git a/sync/cli/Parameters.py b/sync/cli/Parameters.py index bf671b2..15f84e7 100644 --- a/sync/cli/Parameters.py +++ b/sync/cli/Parameters.py @@ -1,4 +1,3 @@ -from argparse import ArgumentParser as ArgumentParserBase # noinspection PyUnresolvedReferences,PyProtectedMember from argparse import ( Action, @@ -7,6 +6,7 @@ _HelpAction, _StoreAction ) +from argparse import ArgumentParser as ArgumentParserBase from pathlib import Path from typing import Sequence, Optional diff --git a/sync/core/Pull.py b/sync/core/Pull.py index ff49f59..e521e5e 100644 --- a/sync/core/Pull.py +++ b/sync/core/Pull.py @@ -2,10 +2,10 @@ import shutil from .Config import Config -from ..model import TrackJson, LocalModule, AttrDict, MagiskUpdateJson, OnlineModule +from ..model import LocalModule, AttrDict, MagiskUpdateJson, OnlineModule from ..modifier import Result -from ..utils import Log, HttpUtils, GitUtils from ..track import LocalTracks +from ..utils import Log, HttpUtils, GitUtils class Pull: diff --git a/sync/core/Pull.pyi b/sync/core/Pull.pyi index e960ed0..889b75d 100644 --- a/sync/core/Pull.pyi +++ b/sync/core/Pull.pyi @@ -1,8 +1,8 @@ from pathlib import Path from typing import Optional, Tuple -from ..modifier import Result from ..model import TrackJson, ConfigJson, OnlineModule +from ..modifier import Result from ..utils import Log diff --git a/sync/modifier/Command.py b/sync/modifier/Command.py index 98994ba..d5e10a1 100644 --- a/sync/modifier/Command.py +++ b/sync/modifier/Command.py @@ -1,6 +1,7 @@ import subprocess from pathlib import Path -from typing import Callable, Any, Optional +from typing import Callable, Optional + from .Result import Result diff --git a/sync/utils/GitUtils.py b/sync/utils/GitUtils.py index 6a9683a..d86e86f 100644 --- a/sync/utils/GitUtils.py +++ b/sync/utils/GitUtils.py @@ -3,8 +3,8 @@ import subprocess from datetime import datetime from pathlib import Path -from typing import Optional from subprocess import CalledProcessError +from typing import Optional from dateutil.parser import parse from requests import HTTPError diff --git a/sync/utils/Log.py b/sync/utils/Log.py index 5733c6c..7a0b838 100644 --- a/sync/utils/Log.py +++ b/sync/utils/Log.py @@ -1,11 +1,11 @@ +import functools +import logging import os import sys -import logging -import functools -from typing import Optional +from datetime import date from glob import glob from pathlib import Path -from datetime import date +from typing import Optional logger_initialized = {} From a6d8a04f07f5b566ff00f5e9f76a0f7d782f9d43 Mon Sep 17 00:00:00 2001 From: ya0211 Date: Wed, 14 Jun 2023 16:51:37 +0800 Subject: [PATCH 5/6] Remove unused class properties --- sync/core/Sync.py | 13 ------------- sync/core/Sync.pyi | 4 ---- 2 files changed, 17 deletions(-) diff --git a/sync/core/Sync.py b/sync/core/Sync.py index 8909f3f..79e24a7 100644 --- a/sync/core/Sync.py +++ b/sync/core/Sync.py @@ -15,7 +15,6 @@ def __init__(self, root_folder, config, tracks=None): self._root_folder = root_folder self._pull = Pull(root_folder, config) - self._is_updatable = False self._is_full_update = True self._json_folder = Config.get_json_folder(root_folder) self._modules_folder = self._pull.modules_folder @@ -31,10 +30,6 @@ def __init__(self, root_folder, config, tracks=None): def __del__(self): self._log.d("__del__") - def _set_updatable(self): - if not self.is_updatable: - self._is_updatable = True - def _check_ids(self, track, online_module): if track.id == online_module.id: return @@ -163,7 +158,6 @@ def update_by_ids(self, module_ids, force, **kwargs): if online_module is None: continue - self._set_updatable() self._log.i(f"update_by_ids: [{track.id}] -> update to {online_module.version_display}") def update_all(self, force, **kwargs): @@ -215,9 +209,6 @@ def create_modules_json(self, to_file): return modules_json def push_by_git(self, branch): - if not self.is_updatable: - return - json_file = self._json_folder.joinpath(ModulesJson.filename()) if not json_file.exists(): self._log.e(f"push_by_git: {json_file.name} is not in {self._json_folder}") @@ -233,7 +224,3 @@ def push_by_git(self, branch): subprocess.run(["git", "add", "."], cwd=self._root_folder.as_posix()) subprocess.run(["git", "commit", "-m", msg], cwd=self._root_folder.as_posix()) subprocess.run(["git", "push", "-u", "origin", branch], cwd=self._root_folder.as_posix()) - - @property - def is_updatable(self): - return self._is_updatable diff --git a/sync/core/Sync.pyi b/sync/core/Sync.pyi index 447c95e..72d04ed 100644 --- a/sync/core/Sync.pyi +++ b/sync/core/Sync.pyi @@ -12,7 +12,6 @@ class Sync: _root_folder: Path _pull: Pull - _is_updatable: bool _is_full_update: bool _json_folder: Path _modules_folder: Path @@ -20,7 +19,6 @@ class Sync: _tracks: BaseTracks def __init__(self, root_folder: Path, config: ConfigJson, tracks: Optional[BaseTracks] = ...): ... - def _set_updatable(self): ... def _check_ids(self, track: TrackJson, online_module: OnlineModule): ... def _update_jsons(self, track: TrackJson, force: bool) -> Optional[OnlineModule]: ... def _check_tracks(self, obj: BaseTracks, cls: type): ... @@ -31,5 +29,3 @@ class Sync: def update_all(self, force: bool, **kwargs): ... def create_modules_json(self, to_file: bool) -> ModulesJson: ... def push_by_git(self, branch: str): ... - @property - def is_updatable(self): ... From 0ca0d1f494fc1b63ec4063633d403a5eb8f69c0e Mon Sep 17 00:00:00 2001 From: ya0211 Date: Fri, 16 Jun 2023 16:23:48 +0800 Subject: [PATCH 6/6] Add metadata to modules.json --- sync/core/Sync.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sync/core/Sync.py b/sync/core/Sync.py index 79e24a7..5bd6ebd 100644 --- a/sync/core/Sync.py +++ b/sync/core/Sync.py @@ -7,6 +7,7 @@ from ..model import ModulesJson, UpdateJson, TrackJson, LocalModule, AttrDict from ..track import BaseTracks, LocalTracks, GithubTracks from ..utils import Log, GitUtils +from ..__version__ import version, versionCode class Sync: @@ -170,6 +171,10 @@ def create_modules_json(self, to_file): modules_json = ModulesJson( name=self._config.repo_name, timestamp=timestamp, + metadata=AttrDict( + version=version, + versionCode=versionCode + ), modules=list() )