From 8f0318dd6bb4ff12ad3adf9f912d7643e0db7c22 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 17 Jun 2024 10:43:01 -0500 Subject: [PATCH 01/16] reckless: remove mimetype The python mimetype package wasn't useful enough in indentifying filetypes anyhow. --- tools/reckless | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tools/reckless b/tools/reckless index ac6247673b96..0957a6f39226 100755 --- a/tools/reckless +++ b/tools/reckless @@ -52,13 +52,12 @@ class Installer: The identification of a plugin language, compiler or interpreter availability, and the install procedures. ''' - def __init__(self, name: str, mimetype: str, + def __init__(self, name: str, exe: Union[str, None] = None, compiler: Union[str, None] = None, manager: Union[str, None] = None, entry: Union[str, None] = None): self.name = name - self.mimetype = mimetype self.entries = [] if entry: self.entries.append(entry) @@ -69,7 +68,7 @@ class Installer: self.dependency_call = None def __repr__(self): - return (f'') def executable(self) -> bool: @@ -840,21 +839,21 @@ def install_to_python_virtual_environment(cloned_plugin: InstInfo): return cloned_plugin -python3venv = Installer('python3venv', 'text/x-python', exe='python3', +python3venv = Installer('python3venv', exe='python3', manager='pip', entry='{name}.py') python3venv.add_entrypoint('{name}') python3venv.add_entrypoint('__init__.py') python3venv.add_dependency_file('requirements.txt') python3venv.dependency_call = install_to_python_virtual_environment -poetryvenv = Installer('poetryvenv', 'text/x-python', exe='python3', +poetryvenv = Installer('poetryvenv', exe='python3', manager='poetry', entry='{name}.py') poetryvenv.add_entrypoint('{name}') poetryvenv.add_entrypoint('__init__.py') poetryvenv.add_dependency_file('pyproject.toml') poetryvenv.dependency_call = install_to_python_virtual_environment -pyprojectViaPip = Installer('pyprojectViaPip', 'text/x-python', exe='python3', +pyprojectViaPip = Installer('pyprojectViaPip', exe='python3', manager='pip', entry='{name}.py') pyprojectViaPip.add_entrypoint('{name}') pyprojectViaPip.add_entrypoint('__init__.py') @@ -863,7 +862,7 @@ pyprojectViaPip.dependency_call = install_to_python_virtual_environment # Nodejs plugin installer -nodejs = Installer('nodejs', 'application/javascript', exe='node', +nodejs = Installer('nodejs', exe='node', manager='npm', entry='{name}.js') nodejs.add_entrypoint('{name}') nodejs.add_dependency_call(['npm', 'install', '--omit=dev']) From 024b6ca99eac7d9be6c9afa382c43bc5b73d11c1 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 1 Jul 2024 14:18:15 -0500 Subject: [PATCH 02/16] reckless: add type hints for InstInfo --- tools/reckless | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/reckless b/tools/reckless index 0957a6f39226..6d2389d583bd 100755 --- a/tools/reckless +++ b/tools/reckless @@ -122,12 +122,12 @@ class InstInfo: def __init__(self, name: str, location: str, git_url: str): self.name = name self.source_loc = str(location) # Used for 'git clone' - self.git_url = git_url # API access for github repos - self.srctype = Source.get_type(location) - self.entry = None # relative to source_loc or subdir - self.deps = None - self.subdir = None - self.commit = None + self.git_url: str = git_url # API access for github repos + self.srctype: Source = Source.get_type(location) + self.entry: SourceFile = None # relative to source_loc or subdir + self.deps: str = None + self.subdir: str = None + self.commit: str = None def __repr__(self): return (f'InstInfo({self.name}, {self.source_loc}, {self.git_url}, ' From e015098bf67ba7668b67bcdc7388a1ba82ed8b6c Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 20 Jun 2024 12:07:09 -0500 Subject: [PATCH 03/16] reckless: place source code in source/plugin_name/ rather than just source/. This is required for cargo install, so let's just use this paradigm globally. --- tools/reckless | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/tools/reckless b/tools/reckless index 6d2389d583bd..8830d9e660cb 100755 --- a/tools/reckless +++ b/tools/reckless @@ -745,12 +745,8 @@ def create_python3_venv(staged_plugin: InstInfo) -> InstInfo: "Create a virtual environment, install dependencies and test plugin." env_path = Path('.venv') env_path_full = Path(staged_plugin.source_loc) / env_path - plugin_path = Path(staged_plugin.source_loc) / 'source' - - # subdir should always be None at this point - if staged_plugin.subdir: - logging.warning("cloned plugin contains subdirectory") - plugin_path = plugin_path / staged_plugin.subdir + assert staged_plugin.subdir # relative dir of original source + plugin_path = Path(staged_plugin.source_loc) / staged_plugin.subdir if shutil.which('poetry') and staged_plugin.deps == 'pyproject.toml': logging.debug('configuring a python virtual environment (poetry) in ' @@ -812,8 +808,10 @@ def create_wrapper(plugin: InstInfo): wrapper.write((f"#!{venv_full_path}/bin/python\n" "import sys\n" "import runpy\n\n" - f"if '{plugin.source_loc}/source' not in sys.path:\n" - f" sys.path.append('{plugin.source_loc}/source')\n" + f"if '{plugin.source_loc}/{plugin.subdir}' not in " + "sys.path:\n" + f" sys.path.append('{plugin.source_loc}/" + f"{plugin.subdir}')\n" f"if '{plugin.source_loc}' in sys.path:\n" f" sys.path.remove('{plugin.source_loc}')\n" f"runpy.run_module(\"{plugin.name}\", " @@ -826,10 +824,6 @@ def install_to_python_virtual_environment(cloned_plugin: InstInfo): '''Called during install in place of a subprocess.run list''' # Delete symlink so that a venv wrapper can take it's place (Path(cloned_plugin.source_loc) / cloned_plugin.entry).unlink() - # The original entrypoint is imported as a python module - ensure - # it has a .py extension. The wrapper can keep the original naming. - entry = Path(cloned_plugin.source_loc) / 'source' / cloned_plugin.entry - entry.rename(entry.with_suffix('.py')) create_python3_venv(cloned_plugin) if not hasattr(cloned_plugin, 'venv'): raise InstallationFailure @@ -1099,17 +1093,28 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: cloned_src.source_loc = plugin_path # Relocate plugin to a staging directory prior to testing - staging_path = inst_path / 'source' + if not Path(inst_path).exists(): + logging.debug(f'creating {inst_path}') + create_dir(inst_path) + if not Path(inst_path / 'source').exists(): + logging.debug(f'creating {inst_path / "source"}') + create_dir(inst_path / 'source') + staging_path = inst_path / 'source' / src.name + logging.debug(f'copying {plugin_path} tree to {staging_path}') shutil.copytree(str(plugin_path), staging_path) staged_src = cloned_src # Because the source files are copied to a 'source' directory, the # get_inst_details function no longer works. (dir must match plugin name) # Set these manually instead. - staged_src.source_loc = str(staging_path.parent) + staged_src.source_loc = str(inst_path) staged_src.srctype = Source.DIRECTORY - staged_src.subdir = None + # Use subdir to redirect the symlink to the actual executable location + staged_src.subdir = f'source/{src.name}' # Create symlink in staging tree to redirect to the plugins entrypoint - Path(staging_path.parent / cloned_src.entry).\ + logging.debug(f"linking source {staging_path / cloned_src.entry} to " + f"{Path(staged_src.source_loc) / cloned_src.entry}") + logging.debug(staged_src) + (Path(staged_src.source_loc) / cloned_src.entry).\ symlink_to(staging_path / cloned_src.entry) # try it out @@ -1139,6 +1144,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: remove_dir(clone_path) remove_dir(inst_path) return None + staged_src.subdir = None test_log = [] try: test = run([Path(staged_src.source_loc).joinpath(staged_src.entry)], From 91d014f9b99c9489d10336ec82d7e6242fd0c595 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 20 Jun 2024 12:59:21 -0500 Subject: [PATCH 04/16] reckless: add installer for rust plugins This follows the same structure that enables python virtual environments: reckless/ / source/ / Changelog-Added: Reckless: added the ability to install rust plugins. --- tools/reckless | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/tools/reckless b/tools/reckless index 8830d9e660cb..0b4d9a3d88b6 100755 --- a/tools/reckless +++ b/tools/reckless @@ -833,6 +833,44 @@ def install_to_python_virtual_environment(cloned_plugin: InstInfo): return cloned_plugin +def cargo_installation(cloned_plugin: InstInfo): + call = ['cargo', 'build', '--release', '-vv'] + # FIXME: the symlinked Cargo.toml allows the installer to identify a valid + # plugin directory, but is unneeded, and actually confuses cargo if not + # removed prior to installing. + cargo_toml_path = Path(cloned_plugin.source_loc) / 'Cargo.toml' + if cargo_toml_path.exists(): + cargo_toml_path.unlink() + + # source_loc now contains a symlink to the entrypoint and 'source/plugin/' + source = Path(cloned_plugin.source_loc) / 'source' / cloned_plugin.name + logging.debug(f'cargo installing from {source}') + if logging.root.level < logging.INFO and not log.capture: + cargo = run(call, cwd=str(source), text=True) + else: + cargo = Popen(call, cwd=str(source), stdout=PIPE, + stderr=PIPE, text=True) + cargo.wait() + + if cargo.returncode == 0: + logging.debug('rust project compiled successfully') + else: + logging.error(cargo.stderr if cargo.stderr else + 'error encountered during build, cargo exited with return ' + f'code {cargo.returncode}') + + logging.debug(f'removing {cloned_plugin.source_loc}') + remove_dir(cloned_plugin.source_loc) + raise InstallationFailure + + # We do need to symlink to the executable binary though. + (Path(cloned_plugin.source_loc) / cloned_plugin.name).\ + symlink_to(source / f'target/release/{cloned_plugin.name}') + cloned_plugin.entry = cloned_plugin.name + + return cloned_plugin + + python3venv = Installer('python3venv', exe='python3', manager='pip', entry='{name}.py') python3venv.add_entrypoint('{name}') @@ -862,7 +900,12 @@ nodejs.add_entrypoint('{name}') nodejs.add_dependency_call(['npm', 'install', '--omit=dev']) nodejs.add_dependency_file('package.json') -INSTALLERS = [python3venv, poetryvenv, pyprojectViaPip, nodejs] +# This entrypoint is used to identify a candidate directory, don't call it. +rust_cargo = Installer('rust', manager='cargo', entry='Cargo.toml') +rust_cargo.add_dependency_file('Cargo.toml') +rust_cargo.dependency_call = cargo_installation + +INSTALLERS = [python3venv, poetryvenv, pyprojectViaPip, nodejs, rust_cargo] def help_alias(targets: list): @@ -911,7 +954,7 @@ def _git_clone(src: InstInfo, dest: Union[PosixPath, str]) -> bool: else: return False git = run(['git', 'clone', '--recurse-submodules', source, str(dest)], - stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=60) + stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=180) if git.returncode != 0: for line in git.stderr.splitlines(): logging.debug(line) From 640be825a2bf4e7c8da5a58ed851563d1343a5e5 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Fri, 21 Jun 2024 16:37:44 -0500 Subject: [PATCH 05/16] reckless: add version information to reckless --- Makefile | 6 +++++- tools/reckless | 12 +++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index bb3ad7984677..43055ffa0e1d 100644 --- a/Makefile +++ b/Makefile @@ -743,7 +743,11 @@ update-wss-proxy-version: cd plugins/wss-proxy && $(MAKE) upgrade-version update-poetry-lock: - poetry update clnrest wss-proxy pyln-client pyln-proto pyln-testing + poetry update clnrest wss-proxy pyln-client pyln-proto pyln-testing update-reckless-version + +update-reckless-version: + @if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi + @sed -i "s/__VERSION__ = '\([.-z]*\)'/__VERSION__ = '$(NEW_VERSION)'/" tools/reckless update-mocks: $(ALL_TEST_PROGRAMS:%=update-mocks/%.c) diff --git a/tools/reckless b/tools/reckless index 0b4d9a3d88b6..a8c757070427 100755 --- a/tools/reckless +++ b/tools/reckless @@ -21,6 +21,8 @@ from urllib.error import HTTPError import venv +__VERSION__ = '24.08' + logging.basicConfig( level=logging.DEBUG, format='[%(asctime)s] %(levelname)s: %(message)s', @@ -1517,8 +1519,10 @@ if __name__ == '__main__': parser.add_argument('-v', '--verbose', action="store_const", dest="loglevel", const=logging.DEBUG, default=logging.WARNING) + parser.add_argument('-V', '--version', action='store_true', + help='return reckless version and exit') cmd1 = parser.add_subparsers(dest='cmd1', help='command', - required=True) + required=False) install_cmd = cmd1.add_parser('install', help='search for and install a ' 'plugin, then test and activate') @@ -1568,6 +1572,12 @@ if __name__ == '__main__': NETWORK = 'regtest' if args.regtest else 'bitcoin' SUPPORTED_NETWORKS = ['bitcoin', 'regtest', 'liquid', 'liquid-regtest', 'litecoin', 'signet', 'testnet'] + if args.version: + print(__VERSION__) + sys.exit(0) + elif args.cmd1 is None: + parser.print_help(sys.stdout) + sys.exit(1) if args.network: if args.network in SUPPORTED_NETWORKS: NETWORK = args.network From 3c26cbe2bcaa3b0818fdd7a96dbd294539beb3c3 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 24 Jun 2024 14:09:31 -0500 Subject: [PATCH 06/16] reckless: make options flags position independent Changelog-Changed: Reckless option flags are now position independent. --- tools/reckless | 88 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 22 deletions(-) diff --git a/tools/reckless b/tools/reckless index a8c757070427..599dc20e2f0a 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1498,29 +1498,43 @@ def list_source(): print(src) +class StoreIdempotent(argparse.Action): + """Make the option idempotent. This adds a secondary argument that doesn't + get reinitialized. The downside is it""" + def __init__(self, option_strings, dest, nargs=None, **kwargs): + super().__init__(option_strings, dest, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + if option_string: + setattr(namespace, self.dest, values) + setattr(namespace, f'{self.dest}_idempotent', values) + + +class StoreTrueIdempotent(argparse._StoreConstAction): + """Make the option idempotent""" + def __init__(self, option_strings, dest, default=False, + required=False, nargs=None, const=None, help=None): + super().__init__(option_strings=option_strings, dest=dest, + const=const, help=help) + + def __call__(self, parser, namespace, values, option_string=None): + if option_string: + setattr(namespace, self.dest, True) + setattr(namespace, f'{self.dest}_idempotent', True) + + +def process_idempotent_args(args): + """Swap idempotently set arguments back in for the default arg names.""" + original_args = dict(vars(args)) + for arg, value in original_args.items(): + if f"{arg}_idempotent" in vars(args): + setattr(args, f"{arg}", vars(args)[f"{arg}_idempotent"]) + delattr(args, f"{arg}_idempotent") + return args + + if __name__ == '__main__': parser = argparse.ArgumentParser() - # This default depends on the .lightning directory - parser.add_argument('-d', '--reckless-dir', - help='specify a data directory for reckless to use', - type=str, default=None) - parser.add_argument('-l', '--lightning', - help='lightning data directory (default:~/.lightning)', - type=str, - default=Path.home().joinpath('.lightning')) - parser.add_argument('-c', '--conf', - help=' config file used by lightningd', - type=str, - default=None) - parser.add_argument('-r', '--regtest', action='store_true') - parser.add_argument('--network', - help="specify a network to use (default: bitcoin)", - type=str) - parser.add_argument('-v', '--verbose', action="store_const", - dest="loglevel", const=logging.DEBUG, - default=logging.WARNING) - parser.add_argument('-V', '--version', action='store_true', - help='return reckless version and exit') cmd1 = parser.add_subparsers(dest='cmd1', help='command', required=False) @@ -1567,7 +1581,38 @@ if __name__ == '__main__': help_cmd.add_argument('targets', type=str, nargs='*') help_cmd.set_defaults(func=help_alias) + all_parsers = [parser, install_cmd, uninstall_cmd, search_cmd, enable_cmd, + disable_cmd, list_parse, source_add, source_rem, help_cmd] + for p in all_parsers: + # This default depends on the .lightning directory + p.add_argument('-d', '--reckless-dir', action=StoreIdempotent, + help='specify a data directory for reckless to use', + type=str, default=None) + p.add_argument('-l', '--lightning', type=str, action=StoreIdempotent, + help='lightning data directory ' + '(default:~/.lightning)', + default=Path.home().joinpath('.lightning')) + p.add_argument('-c', '--conf', action=StoreIdempotent, + help=' config file used by lightningd', + type=str, + default=None) + p.add_argument('-r', '--regtest', action=StoreTrueIdempotent) + p.add_argument('--network', action=StoreIdempotent, + help="specify a network to use (default: bitcoin)", + type=str) + p.add_argument('-v', '--verbose', action=StoreTrueIdempotent, + const=None) + p.add_argument('-V', '--version', action='store_true', + help='return reckless version and exit') + p.add_argument('-m', '--machine', action='store_true') + args = parser.parse_args() + args = process_idempotent_args(args) + + if args.verbose: + logging.root.setLevel(logging.DEBUG) + else: + logging.root.setLevel(logging.WARNING) NETWORK = 'regtest' if args.regtest else 'bitcoin' SUPPORTED_NETWORKS = ['bitcoin', 'regtest', 'liquid', 'liquid-regtest', @@ -1607,7 +1652,6 @@ if __name__ == '__main__': API_GITHUB_COM = os.environ['REDIR_GITHUB_API'] if 'REDIR_GITHUB' in os.environ: GITHUB_COM = os.environ['REDIR_GITHUB'] - logging.root.setLevel(args.loglevel) GITHUB_API_FALLBACK = False if 'GITHUB_API_FALLBACK' in os.environ: From 24f07338e5de276e03d906271763546f67d2287f Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 1 Jul 2024 14:19:18 -0500 Subject: [PATCH 07/16] reckless: add logger class This will allow redirection of json output in the following commits. --- tools/reckless | 195 ++++++++++++++++++++++++++++--------------------- 1 file changed, 112 insertions(+), 83 deletions(-) diff --git a/tools/reckless b/tools/reckless index 599dc20e2f0a..c81cce8d5b5e 100755 --- a/tools/reckless +++ b/tools/reckless @@ -30,6 +30,34 @@ logging.basicConfig( ) +class Logger: + """Redirect logging output to a json object or stdout as appropriate.""" + def __init__(self, capture: bool = False): + self.json_output = {"result": None, + "log": []} + self.capture = capture + + def debug(self, to_log: str): + assert isinstance(to_log, str) or hasattr(to_log, "__repr__") + if logging.root.level > logging.DEBUG: + return + if self.capture: + self.json_output['log'].append(to_log) + else: + logging.debug(to_log) + + def warning(self, to_log: str): + assert isinstance(to_log, str) or hasattr(to_log, "__repr__") + if logging.root.level > logging.WARNING: + return + if self.capture: + self.json_output['log'].append(to_log) + else: + logging.warning(to_log) + + +log = Logger() + repos = ['https://github.com/lightningd/plugins'] @@ -186,7 +214,7 @@ class InstInfo: self.entry = found_entry.name self.deps = found_dep.name return sub - logging.debug(f"missing dependency for {self}") + log.debug(f"missing dependency for {self}") found_entry = None for file in sub.contents: if isinstance(file, SourceDir): @@ -207,9 +235,9 @@ class InstInfo: # clone source to reckless dir target = copy_remote_git_source(self) if not target: - logging.warning(f"could not clone github source {self}") + log.warning(f"could not clone github source {self}") return False - logging.debug(f"falling back to cloning remote repo {self}") + log.debug(f"falling back to cloning remote repo {self}") # Update to reflect use of a local clone self.source_loc = target.location self.srctype = target.srctype @@ -247,9 +275,10 @@ def remove_dir(directory: str) -> bool: shutil.rmtree(directory) return True except NotADirectoryError: - print(f"Tried to remove directory {directory} that does not exist.") + log.warning(f"Tried to remove directory {directory} that " + "does not exist.") except PermissionError: - print(f"Permission denied removing dir: {directory}") + log.warning(f"Permission denied removing dir: {directory}") return False @@ -310,7 +339,6 @@ class SourceDir(): return if not self.srctype: self.srctype = Source.get_type(self.location) - # logging.debug(f"populating {self.srctype} {self.location}") if self.srctype == Source.DIRECTORY: self.contents = populate_local_dir(self.location) elif self.srctype in [Source.LOCAL_REPO, Source.GIT_LOCAL_CLONE]: @@ -395,7 +423,7 @@ def populate_local_repo(path: str, parent=None) -> list: This populates all intermediate directories and the file.""" parentdir = parent if mypath == '.': - logging.debug(' asked to populate root dir') + log.debug(' asked to populate root dir') return # reverse the parents pdirs = mypath @@ -432,7 +460,7 @@ def populate_local_repo(path: str, parent=None) -> list: proc = run(['git', '-C', path, 'submodule', 'status'], stdout=PIPE, stderr=PIPE, text=True, timeout=5) if proc.returncode != 0: - logging.debug(f"'git submodule status' of repo {path} failed") + log.debug(f"'git submodule status' of repo {path} failed") return None submodules = [] for sub in proc.stdout.splitlines(): @@ -444,7 +472,7 @@ def populate_local_repo(path: str, parent=None) -> list: '--name-only', ver] proc = run(git_call, stdout=PIPE, stderr=PIPE, text=True, timeout=5) if proc.returncode != 0: - logging.debug(f'ls-tree of repo {path} failed') + log.debug(f'ls-tree of repo {path} failed') return None for filepath in proc.stdout.splitlines(): @@ -483,7 +511,7 @@ def source_element_from_repo_api(member: dict): # git_url with /tree/ presents results a little differently elif 'type' in member and 'path' in member and 'url' in member: if member['type'] not in ['tree', 'blob']: - logging.debug(f' skipping {member["path"]} type={member["type"]}') + log.debug(f' skipping {member["path"]} type={member["type"]}') if member['type'] == 'tree': return SourceDir(member['url'], srctype=Source.GITHUB_REPO, name=member['path']) @@ -532,7 +560,7 @@ def populate_github_repo(url: str) -> list: git_url = api_url if "api.github.com" in git_url: # This lets us redirect to handle blackbox testing - logging.debug(f'fetching from gh API: {git_url}') + log.debug(f'fetching from gh API: {git_url}') git_url = (API_GITHUB_COM + git_url.split("api.github.com")[-1]) # Ratelimiting occurs for non-authenticated GH API calls at 60 in 1 hour. r = urlopen(git_url, timeout=5) @@ -553,14 +581,14 @@ def copy_remote_git_source(github_source: InstInfo): """clone or fetch & checkout a local copy of a remote git repo""" user, repo = Source.get_github_user_repo(github_source.source_loc) if not user or not repo: - logging.warning('could not extract github user and repo ' - f'name for {github_source.source_loc}') + log.warning('could not extract github user and repo ' + f'name for {github_source.source_loc}') return None local_path = RECKLESS_DIR / '.remote_sources' / user create_dir(RECKLESS_DIR / '.remote_sources') if not create_dir(local_path): - logging.warning(f'could not provision dir {local_path} to ' - f'clone remote source {github_source.source_loc}') + log.warning(f'could not provision dir {local_path} to ' + f'clone remote source {github_source.source_loc}') return None local_path = local_path / repo if local_path.exists(): @@ -602,8 +630,8 @@ class Config(): # FIXME: Handle write failure return default_text else: - logging.debug('could not create the parent directory ' + - parent_path) + log.debug('could not create the parent directory ' + + parent_path) raise FileNotFoundError('invalid parent directory') def editConfigFile(self, addline: Union[str, None], @@ -751,8 +779,8 @@ def create_python3_venv(staged_plugin: InstInfo) -> InstInfo: plugin_path = Path(staged_plugin.source_loc) / staged_plugin.subdir if shutil.which('poetry') and staged_plugin.deps == 'pyproject.toml': - logging.debug('configuring a python virtual environment (poetry) in ' - f'{env_path_full}') + log.debug('configuring a python virtual environment (poetry) in ' + f'{env_path_full}') # The virtual environment should be located with the plugin. # This installs it to .venv instead of in the global location. mod_poetry_env = os.environ @@ -779,9 +807,9 @@ def create_python3_venv(staged_plugin: InstInfo) -> InstInfo: else: builder = venv.EnvBuilder(with_pip=True) builder.create(env_path_full) - logging.debug('configuring a python virtual environment (pip) in ' - f'{env_path_full}') - logging.debug(f'virtual environment created in {env_path_full}.') + log.debug('configuring a python virtual environment (pip) in ' + f'{env_path_full}') + log.debug(f'virtual environment created in {env_path_full}.') if staged_plugin.deps == 'pyproject.toml': pip = run(['bin/pip', 'install', str(plugin_path)], check=False, cwd=plugin_path) @@ -790,9 +818,9 @@ def create_python3_venv(staged_plugin: InstInfo) -> InstInfo: str(plugin_path / 'requirements.txt')], check=False, cwd=plugin_path) else: - logging.debug("no python dependency file") + log.debug("no python dependency file") if pip and pip.returncode != 0: - logging.debug("install to virtual environment failed") + log.debug("install to virtual environment failed") print('error encountered installing dependencies') raise InstallationFailure @@ -829,8 +857,8 @@ def install_to_python_virtual_environment(cloned_plugin: InstInfo): create_python3_venv(cloned_plugin) if not hasattr(cloned_plugin, 'venv'): raise InstallationFailure - logging.debug('virtual environment for cloned plugin: ' - f'{cloned_plugin.venv}') + log.debug('virtual environment for cloned plugin: ' + f'{cloned_plugin.venv}') create_wrapper(cloned_plugin) return cloned_plugin @@ -846,22 +874,23 @@ def cargo_installation(cloned_plugin: InstInfo): # source_loc now contains a symlink to the entrypoint and 'source/plugin/' source = Path(cloned_plugin.source_loc) / 'source' / cloned_plugin.name - logging.debug(f'cargo installing from {source}') - if logging.root.level < logging.INFO and not log.capture: - cargo = run(call, cwd=str(source), text=True) + log.debug(f'cargo installing from {source}') + run(['ls'], cwd=str(source), text=True, check=True) + if logging.root.level < logging.WARNING: + cargo = Popen(call, cwd=str(source), text=True) else: cargo = Popen(call, cwd=str(source), stdout=PIPE, stderr=PIPE, text=True) cargo.wait() if cargo.returncode == 0: - logging.debug('rust project compiled successfully') + log.debug('rust project compiled successfully') else: - logging.error(cargo.stderr if cargo.stderr else - 'error encountered during build, cargo exited with return ' - f'code {cargo.returncode}') + log.error(cargo.stderr if cargo.stderr else + 'error encountered during build, cargo exited with return ' + f'code {cargo.returncode}') - logging.debug(f'removing {cloned_plugin.source_loc}') + log.debug(f'removing {cloned_plugin.source_loc}') remove_dir(cloned_plugin.source_loc) raise InstallationFailure @@ -935,8 +964,8 @@ def _source_search(name: str, src: str) -> Union[InstInfo, None]: if local_clone_location.exists(): # Make sure it's the correct remote source and fetch any updates. if _git_update(source, local_clone_location): - logging.debug(f"Using local clone of {src}: " - f"{local_clone_location}") + log.debug(f"Using local clone of {src}: " + f"{local_clone_location}") source.source_loc = local_clone_location source.srctype = Source.GIT_LOCAL_CLONE @@ -959,7 +988,7 @@ def _git_clone(src: InstInfo, dest: Union[PosixPath, str]) -> bool: stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=180) if git.returncode != 0: for line in git.stderr.splitlines(): - logging.debug(line) + log.debug(line) if Path(dest).exists(): remove_dir(str(dest)) print('Error: Failed to clone repo') @@ -993,8 +1022,8 @@ def _git_update(github_source: InstInfo, local_copy: PosixPath): return False default_branch = git.stdout.splitlines()[0] if default_branch != 'origin/master': - logging.debug(f'UNUSUAL: fetched default branch {default_branch} for ' - f'{github_source.source_loc}') + log.debug(f'UNUSUAL: fetched default branch {default_branch} for ' + f'{github_source.source_loc}') # Checkout default branch git = run(['git', 'checkout', default_branch], @@ -1040,7 +1069,7 @@ def _checkout_commit(orig_src: InstInfo, if orig_src.srctype in [Source.LOCAL_REPO, Source.GITHUB_REPO, Source.OTHER_URL, Source.GIT_LOCAL_CLONE]: if orig_src.commit: - logging.debug(f"Checking out {orig_src.commit}") + log.debug(f"Checking out {orig_src.commit}") checkout = Popen(['git', 'checkout', orig_src.commit], cwd=str(cloned_path), stdout=PIPE, stderr=PIPE) @@ -1050,21 +1079,21 @@ def _checkout_commit(orig_src: InstInfo, f'commit {orig_src.commit}') return None else: - logging.debug("using latest commit of default branch") + log.debug("using latest commit of default branch") # Log the commit we actually used (for installation metadata) git = run(['git', 'rev-parse', 'HEAD'], cwd=str(cloned_path), stdout=PIPE, stderr=PIPE, text=True, check=False, timeout=60) if git.returncode == 0: head_commit = git.stdout.splitlines()[0] - logging.debug(f'checked out HEAD: {head_commit}') + log.debug(f'checked out HEAD: {head_commit}') cloned_src.commit = head_commit else: - logging.debug(f'unable to collect commit: {git.stderr}') + log.debug(f'unable to collect commit: {git.stderr}') else: if orig_src.commit: - logging.warning("unable to checkout commit/tag on non-repository " - "source") + log.warning("unable to checkout commit/tag on non-repository " + "source") return cloned_path if cloned_src.subdir is not None: @@ -1074,7 +1103,7 @@ def _checkout_commit(orig_src: InstInfo, def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: """make sure the repo exists and clone it.""" - logging.debug(f'Install requested from {src}.') + log.debug(f'Install requested from {src}.') if RECKLESS_CONFIG is None: print('error: reckless install directory unavailable') sys.exit(2) @@ -1082,21 +1111,21 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: # Use a unique directory for each cloned repo. tmp_path = get_temp_reckless_dir() if not create_dir(tmp_path): - logging.debug(f'failed to create {tmp_path}') + log.debug(f'failed to create {tmp_path}') return None clone_path = tmp_path / 'clone' if not create_dir(tmp_path): - logging.debug(f'failed to create {clone_path}') + log.debug(f'failed to create {clone_path}') return None # we rename the original repo here. plugin_path = clone_path / src.name inst_path = Path(RECKLESS_CONFIG.reckless_dir) / src.name if Path(clone_path).exists(): - logging.debug(f'{clone_path} already exists - deleting') + log.debug(f'{clone_path} already exists - deleting') shutil.rmtree(clone_path) if src.srctype == Source.DIRECTORY: - logging.debug(("copying local directory contents from" - f" {src.source_loc}")) + log.debug(("copying local directory contents from" + f" {src.source_loc}")) create_dir(clone_path) shutil.copytree(src.source_loc, plugin_path) elif src.srctype in [Source.LOCAL_REPO, Source.GITHUB_REPO, @@ -1108,9 +1137,9 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: # Depending on how we accessed the original source, there may be install # details missing. Searching the cloned repo makes sure we have it. cloned_src = _source_search(src.name, str(clone_path)) - logging.debug(f'cloned_src: {cloned_src}') + log.debug(f'cloned_src: {cloned_src}') if not cloned_src: - logging.debug('failed to find plugin after cloning repo.') + log.debug('failed to find plugin after cloning repo.') return None # If a specific commit or tag was requested, check it out now. @@ -1126,11 +1155,11 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: if inst_method.dependency_file is not None: if inst_method.dependency_file not in os.listdir(plugin_path): continue - logging.debug(f"using installer {inst_method.name}") + log.debug(f"using installer {inst_method.name}") INSTALLER = inst_method break if not INSTALLER: - logging.debug('Could not find a suitable installer method.') + log.debug('Could not find a suitable installer method.') return None if not cloned_src.entry: # The plugin entrypoint may not be discernable prior to cloning. @@ -1139,13 +1168,13 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: # Relocate plugin to a staging directory prior to testing if not Path(inst_path).exists(): - logging.debug(f'creating {inst_path}') + log.debug(f'creating {inst_path}') create_dir(inst_path) if not Path(inst_path / 'source').exists(): - logging.debug(f'creating {inst_path / "source"}') + log.debug(f'creating {inst_path / "source"}') create_dir(inst_path / 'source') staging_path = inst_path / 'source' / src.name - logging.debug(f'copying {plugin_path} tree to {staging_path}') + log.debug(f'copying {plugin_path} tree to {staging_path}') shutil.copytree(str(plugin_path), staging_path) staged_src = cloned_src # Because the source files are copied to a 'source' directory, the @@ -1156,9 +1185,9 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: # Use subdir to redirect the symlink to the actual executable location staged_src.subdir = f'source/{src.name}' # Create symlink in staging tree to redirect to the plugins entrypoint - logging.debug(f"linking source {staging_path / cloned_src.entry} to " - f"{Path(staged_src.source_loc) / cloned_src.entry}") - logging.debug(staged_src) + log.debug(f"linking source {staging_path / cloned_src.entry} to " + f"{Path(staged_src.source_loc) / cloned_src.entry}") + log.debug(staged_src) (Path(staged_src.source_loc) / cloned_src.entry).\ symlink_to(staging_path / cloned_src.entry) @@ -1171,7 +1200,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: return None else: for call in INSTALLER.dependency_call: - logging.debug(f"Install: invoking '{' '.join(call)}'") + log.debug(f"Install: invoking '{' '.join(call)}'") if logging.root.level < logging.WARNING: pip = Popen(call, cwd=staging_path, text=True) else: @@ -1185,7 +1214,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: else: print('error encountered installing dependencies') if pip.stdout: - logging.debug(pip.stdout.read()) + log.debug(pip.stdout.read()) remove_dir(clone_path) remove_dir(inst_path) return None @@ -1202,9 +1231,9 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: # If the plugin is still running, it's assumed to be okay. returncode = 0 if returncode != 0: - logging.debug("plugin testing error:") + log.debug("plugin testing error:") for line in test_log: - logging.debug(f' {line}') + log.debug(f' {line}') print('plugin testing failed') remove_dir(clone_path) remove_dir(inst_path) @@ -1221,16 +1250,16 @@ def install(plugin_name: str): assert isinstance(plugin_name, str) # Specify a tag or commit to checkout by adding @ to plugin name if '@' in plugin_name: - logging.debug("testing for a commit/tag in plugin name") + log.debug("testing for a commit/tag in plugin name") name, commit = plugin_name.split('@', 1) else: name = plugin_name commit = None - logging.debug(f"Searching for {name}") + log.debug(f"Searching for {name}") src = search(name) if src: src.commit = commit - logging.debug(f'Retrieving {src.name} from {src.source_loc}') + log.debug(f'Retrieving {src.name} from {src.source_loc}') installed = _install_plugin(src) if not installed: print('installation aborted') @@ -1252,13 +1281,13 @@ def install(plugin_name: str): def uninstall(plugin_name: str): """disables plugin and deletes the plugin's reckless dir""" assert isinstance(plugin_name, str) - logging.debug(f'Uninstalling plugin {plugin_name}') + log.debug(f'Uninstalling plugin {plugin_name}') disable(plugin_name) inst = InferInstall(plugin_name) if not Path(inst.entry).exists(): print(f'cannot find installed plugin at expected path {inst.entry}') sys.exit(1) - logging.debug(f'looking for {str(Path(inst.entry).parent)}') + log.debug(f'looking for {str(Path(inst.entry).parent)}') if remove_dir(str(Path(inst.entry).parent)): print(f"{inst.name} uninstalled successfully.") @@ -1281,7 +1310,7 @@ def search(plugin_name: str) -> Union[InstInfo, None]: for source in ordered_sources: srctype = Source.get_type(source) if srctype == Source.UNKNOWN: - logging.debug(f'cannot search {srctype} {source}') + log.debug(f'cannot search {srctype} {source}') continue if srctype in [Source.DIRECTORY, Source.LOCAL_REPO, Source.GITHUB_REPO, Source.OTHER_URL]: @@ -1289,11 +1318,11 @@ def search(plugin_name: str) -> Union[InstInfo, None]: if not found: continue print(f"found {found.name} in source: {found.source_loc}") - logging.debug(f"entry: {found.entry}") + log.debug(f"entry: {found.entry}") if found.subdir: - logging.debug(f'sub-directory: {found.subdir}') + log.debug(f'sub-directory: {found.subdir}') return found - logging.debug("Search exhausted all sources") + log.debug("Search exhausted all sources") return None @@ -1348,18 +1377,18 @@ def enable(plugin_name: str): if not Path(path).exists(): print(f'cannot find installed plugin at expected path {path}') sys.exit(1) - logging.debug(f'activating {plugin_name}') + log.debug(f'activating {plugin_name}') try: lightning_cli('plugin', 'start', path) except CLIError as err: if 'already registered' in err.message: - logging.debug(f'{inst.name} is already running') + log.debug(f'{inst.name} is already running') else: print(f'reckless: {inst.name} failed to start!') raise err except RPCError: - logging.debug(('lightningd rpc unavailable. ' - 'Skipping dynamic activation.')) + log.debug(('lightningd rpc unavailable. ' + 'Skipping dynamic activation.')) RECKLESS_CONFIG.enable_plugin(path) print(f'{inst.name} enabled') @@ -1373,18 +1402,18 @@ def disable(plugin_name: str): if not Path(path).exists(): sys.stderr.write(f'Could not find plugin at {path}\n') sys.exit(1) - logging.debug(f'deactivating {plugin_name}') + log.debug(f'deactivating {plugin_name}') try: lightning_cli('plugin', 'stop', path) except CLIError as err: if err.code == -32602: - logging.debug('plugin not currently running') + log.debug('plugin not currently running') else: print('lightning-cli plugin stop failed') raise err except RPCError: - logging.debug(('lightningd rpc unavailable. ' - 'Skipping dynamic deactivation.')) + log.debug(('lightningd rpc unavailable. ' + 'Skipping dynamic deactivation.')) RECKLESS_CONFIG.disable_plugin(path) print(f'{inst.name} disabled') @@ -1454,7 +1483,7 @@ def load_sources() -> list: sources_file = get_sources_file() # This would have been created if possible if not Path(sources_file).exists(): - logging.debug('Warning: Reckless requires write access') + log.debug('Warning: Reckless requires write access') Config(path=str(sources_file), default_text='https://github.com/lightningd/plugins') return ['https://github.com/lightningd/plugins'] From f90561fe175d3ecd18fa559183e10d2d16470e9a Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 2 Jul 2024 07:50:50 -0500 Subject: [PATCH 08/16] reckless: add json output option Also redirect config creation prompts to stderr in order to not interfere with json output on stdout. Changelog-Added: reckless provides json output with option flag -j/--json --- tests/test_reckless.py | 3 +- tools/reckless | 119 ++++++++++++++++++++++++++--------------- 2 files changed, 77 insertions(+), 45 deletions(-) diff --git a/tests/test_reckless.py b/tests/test_reckless.py index bca922e013f7..8f32447b014b 100644 --- a/tests/test_reckless.py +++ b/tests/test_reckless.py @@ -111,7 +111,8 @@ def get_reckless_node(node_factory): def check_stderr(stderr): def output_okay(out): for warning in ['[notice]', 'WARNING:', 'npm WARN', - 'npm notice', 'DEPRECATION:', 'Creating virtualenv']: + 'npm notice', 'DEPRECATION:', 'Creating virtualenv', + 'config file not found:', 'press [Y]']: if out.startswith(warning): return True return False diff --git a/tools/reckless b/tools/reckless index c81cce8d5b5e..8a54d7416e26 100755 --- a/tools/reckless +++ b/tools/reckless @@ -24,7 +24,7 @@ import venv __VERSION__ = '24.08' logging.basicConfig( - level=logging.DEBUG, + level=logging.INFO, format='[%(asctime)s] %(levelname)s: %(message)s', handlers=[logging.StreamHandler(stream=sys.stdout)], ) @@ -33,7 +33,7 @@ logging.basicConfig( class Logger: """Redirect logging output to a json object or stdout as appropriate.""" def __init__(self, capture: bool = False): - self.json_output = {"result": None, + self.json_output = {"result": [], "log": []} self.capture = capture @@ -42,19 +42,38 @@ class Logger: if logging.root.level > logging.DEBUG: return if self.capture: - self.json_output['log'].append(to_log) + self.json_output['log'].append(f"DEBUG: {to_log}") else: logging.debug(to_log) + def info(self, to_log: str): + assert isinstance(to_log, str) or hasattr(to_log, "__repr__") + if logging.root.level > logging.INFO: + return + if self.capture: + self.json_output['log'].append(f"INFO: {to_log}") + self.json_output['result'].append(to_log) + else: + print(to_log) + def warning(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") if logging.root.level > logging.WARNING: return if self.capture: - self.json_output['log'].append(to_log) + self.json_output['log'].append(f"WARNING: {to_log}") else: logging.warning(to_log) + def error(self, to_log: str): + assert isinstance(to_log, str) or hasattr(to_log, "__repr__") + if logging.root.level > logging.ERROR: + return + if self.capture: + self.json_output['log'].append(f"ERROR: {to_log}") + else: + logging.error(to_log) + log = Logger() @@ -615,11 +634,15 @@ class Config(): with open(config_path, 'r+') as f: config_content = f.readlines() return config_content + # redirecting the prompts to stderr is kinder for json consumers + tmp = sys.stdout + sys.stdout = sys.stderr print(f'config file not found: {config_path}') if warn: confirm = input('press [Y] to create one now.\n').upper() == 'Y' else: confirm = True + sys.stdout = tmp if not confirm: sys.exit(1) parent_path = Path(config_path).parent @@ -821,11 +844,11 @@ def create_python3_venv(staged_plugin: InstInfo) -> InstInfo: log.debug("no python dependency file") if pip and pip.returncode != 0: log.debug("install to virtual environment failed") - print('error encountered installing dependencies') + log.error('error encountered installing dependencies') raise InstallationFailure staged_plugin.venv = env_path - print('dependencies installed successfully') + log.info('dependencies installed successfully') return staged_plugin @@ -876,7 +899,7 @@ def cargo_installation(cloned_plugin: InstInfo): source = Path(cloned_plugin.source_loc) / 'source' / cloned_plugin.name log.debug(f'cargo installing from {source}') run(['ls'], cwd=str(source), text=True, check=True) - if logging.root.level < logging.WARNING: + if logging.root.level < logging.INFO: cargo = Popen(call, cwd=str(source), text=True) else: cargo = Popen(call, cwd=str(source), stdout=PIPE, @@ -943,7 +966,7 @@ def help_alias(targets: list): if len(targets) == 0: parser.print_help(sys.stdout) else: - print('try "reckless {} -h"'.format(' '.join(targets))) + log.info('try "reckless {} -h"'.format(' '.join(targets))) sys.exit(1) @@ -975,7 +998,7 @@ def _source_search(name: str, src: str) -> Union[InstInfo, None]: def _git_clone(src: InstInfo, dest: Union[PosixPath, str]) -> bool: - print(f'cloning {src.srctype} {src}') + log.info(f'cloning {src.srctype} {src}') if src.srctype == Source.GITHUB_REPO: assert 'github.com' in src.source_loc source = f"{GITHUB_COM}" + src.source_loc.split("github.com")[-1] @@ -991,7 +1014,7 @@ def _git_clone(src: InstInfo, dest: Union[PosixPath, str]) -> bool: log.debug(line) if Path(dest).exists(): remove_dir(str(dest)) - print('Error: Failed to clone repo') + log.error('Failed to clone repo') return False return True @@ -1075,8 +1098,8 @@ def _checkout_commit(orig_src: InstInfo, stdout=PIPE, stderr=PIPE) checkout.wait() if checkout.returncode != 0: - print('failed to checkout referenced ' - f'commit {orig_src.commit}') + log.warning('failed to checkout referenced ' + f'commit {orig_src.commit}') return None else: log.debug("using latest commit of default branch") @@ -1105,7 +1128,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: """make sure the repo exists and clone it.""" log.debug(f'Install requested from {src}.') if RECKLESS_CONFIG is None: - print('error: reckless install directory unavailable') + log.error('reckless install directory unavailable') sys.exit(2) # Use a unique directory for each cloned repo. @@ -1201,7 +1224,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: else: for call in INSTALLER.dependency_call: log.debug(f"Install: invoking '{' '.join(call)}'") - if logging.root.level < logging.WARNING: + if logging.root.level < logging.INFO: pip = Popen(call, cwd=staging_path, text=True) else: pip = Popen(call, cwd=staging_path, stdout=PIPE, @@ -1210,9 +1233,9 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: # FIXME: handle output of multiple calls if pip.returncode == 0: - print('dependencies installed successfully') + log.info('dependencies installed successfully') else: - print('error encountered installing dependencies') + log.error('error encountered installing dependencies') if pip.stdout: log.debug(pip.stdout.read()) remove_dir(clone_path) @@ -1234,13 +1257,13 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: log.debug("plugin testing error:") for line in test_log: log.debug(f' {line}') - print('plugin testing failed') + log.error('plugin testing failed') remove_dir(clone_path) remove_dir(inst_path) return None add_installation_metadata(staged_src, src) - print(f'plugin installed: {inst_path}') + log.info(f'plugin installed: {inst_path}') remove_dir(clone_path) return staged_src @@ -1262,7 +1285,7 @@ def install(plugin_name: str): log.debug(f'Retrieving {src.name} from {src.source_loc}') installed = _install_plugin(src) if not installed: - print('installation aborted') + log.warning('installation aborted') sys.exit(1) # Match case of the containing directory @@ -1273,8 +1296,8 @@ def install(plugin_name: str): RECKLESS_CONFIG.enable_plugin(inst_path) enable(installed.name) return - print(('dynamic activation failed: ' - f'{installed.name} not found in reckless directory')) + log.error(('dynamic activation failed: ' + f'{installed.name} not found in reckless directory')) sys.exit(1) @@ -1285,11 +1308,12 @@ def uninstall(plugin_name: str): disable(plugin_name) inst = InferInstall(plugin_name) if not Path(inst.entry).exists(): - print(f'cannot find installed plugin at expected path {inst.entry}') + log.error("cannot find installed plugin at expected path" + f"{inst.entry}") sys.exit(1) log.debug(f'looking for {str(Path(inst.entry).parent)}') if remove_dir(str(Path(inst.entry).parent)): - print(f"{inst.name} uninstalled successfully.") + log.info(f"{inst.name} uninstalled successfully.") def search(plugin_name: str) -> Union[InstInfo, None]: @@ -1317,7 +1341,7 @@ def search(plugin_name: str) -> Union[InstInfo, None]: found = _source_search(plugin_name, source) if not found: continue - print(f"found {found.name} in source: {found.source_loc}") + log.info(f"found {found.name} in source: {found.source_loc}") log.debug(f"entry: {found.entry}") if found.subdir: log.debug(f'sub-directory: {found.subdir}') @@ -1375,7 +1399,7 @@ def enable(plugin_name: str): inst = InferInstall(plugin_name) path = inst.entry if not Path(path).exists(): - print(f'cannot find installed plugin at expected path {path}') + log.error(f'cannot find installed plugin at expected path {path}') sys.exit(1) log.debug(f'activating {plugin_name}') try: @@ -1384,13 +1408,13 @@ def enable(plugin_name: str): if 'already registered' in err.message: log.debug(f'{inst.name} is already running') else: - print(f'reckless: {inst.name} failed to start!') + log.error(f'reckless: {inst.name} failed to start!') raise err except RPCError: log.debug(('lightningd rpc unavailable. ' 'Skipping dynamic activation.')) RECKLESS_CONFIG.enable_plugin(path) - print(f'{inst.name} enabled') + log.info(f'{inst.name} enabled') def disable(plugin_name: str): @@ -1409,13 +1433,13 @@ def disable(plugin_name: str): if err.code == -32602: log.debug('plugin not currently running') else: - print('lightning-cli plugin stop failed') + log.error('lightning-cli plugin stop failed') raise err except RPCError: log.debug(('lightningd rpc unavailable. ' 'Skipping dynamic deactivation.')) RECKLESS_CONFIG.disable_plugin(path) - print(f'{inst.name} disabled') + log.info(f'{inst.name} disabled') def load_config(reckless_dir: Union[str, None] = None, @@ -1442,9 +1466,9 @@ def load_config(reckless_dir: Union[str, None] = None, reck_conf_path = Path(reckless_dir) / f'{network}-reckless.conf' if net_conf: if str(network_path) != net_conf.conf_fp: - print('error: reckless configuration does not match lightningd:\n' - f'reckless network config path: {network_path}\n' - f'lightningd active config: {net_conf.conf_fp}') + log.error('reckless configuration does not match lightningd:\n' + f'reckless network config path: {network_path}\n' + f'lightningd active config: {net_conf.conf_fp}') sys.exit(1) else: # The network-specific config file (bitcoin by default) @@ -1453,8 +1477,8 @@ def load_config(reckless_dir: Union[str, None] = None, try: reckless_conf = RecklessConfig(path=reck_conf_path) except FileNotFoundError: - print('Error: reckless config file could not be written: ', - str(reck_conf_path)) + log.error('reckless config file could not be written: ' + + str(reck_conf_path)) sys.exit(1) if not net_conf: print('Error: could not load or create the network specific lightningd' @@ -1506,7 +1530,7 @@ def add_source(src: str): default_text='https://github.com/lightningd/plugins') my_file.editConfigFile(src, None) else: - print(f'failed to add source {src}') + log.warning(f'failed to add source {src}') def remove_source(src: str): @@ -1516,15 +1540,15 @@ def remove_source(src: str): my_file = Config(path=get_sources_file(), default_text='https://github.com/lightningd/plugins') my_file.editConfigFile(None, src) - print('plugin source removed') + log.info('plugin source removed') else: - print(f'source not found: {src}') + log.warning(f'source not found: {src}') def list_source(): """Provide the user with all stored source repositories.""" for src in sources_from_file(): - print(src) + log.info(src) class StoreIdempotent(argparse.Action): @@ -1633,22 +1657,25 @@ if __name__ == '__main__': const=None) p.add_argument('-V', '--version', action='store_true', help='return reckless version and exit') - p.add_argument('-m', '--machine', action='store_true') + p.add_argument('-j', '--json', action=StoreTrueIdempotent, + help='output in json format') args = parser.parse_args() args = process_idempotent_args(args) + if args.json: + log.capture = True + if args.verbose: logging.root.setLevel(logging.DEBUG) else: - logging.root.setLevel(logging.WARNING) + logging.root.setLevel(logging.INFO) NETWORK = 'regtest' if args.regtest else 'bitcoin' SUPPORTED_NETWORKS = ['bitcoin', 'regtest', 'liquid', 'liquid-regtest', 'litecoin', 'signet', 'testnet'] if args.version: - print(__VERSION__) - sys.exit(0) + log.info(__VERSION__) elif args.cmd1 is None: parser.print_help(sys.stdout) sys.exit(1) @@ -1656,7 +1683,7 @@ if __name__ == '__main__': if args.network in SUPPORTED_NETWORKS: NETWORK = args.network else: - print(f"Error: {args.network} network not supported") + log.error(f"{args.network} network not supported") LIGHTNING_DIR = Path(args.lightning) # This env variable is set under CI testing LIGHTNING_CLI_CALL = [os.environ.get('LIGHTNING_CLI')] @@ -1693,5 +1720,9 @@ if __name__ == '__main__': sys.exit(0) for target in args.targets: args.func(target) - else: + elif 'func' in args: args.func() + + # reply with json if requested + if log.capture: + print(json.dumps(log.json_output, indent=4)) From 2aa07aecb9b14017c2b3caaf8098d90beded0248 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 22 Jul 2024 13:42:44 -0500 Subject: [PATCH 09/16] reckless: all command functions return objects to enable json out This more easily allows list output for commands accepting list input, i.e., installing 3 plugins produces 3 outputs. --- tools/reckless | 96 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 30 deletions(-) diff --git a/tools/reckless b/tools/reckless index 8a54d7416e26..a11bd018e818 100755 --- a/tools/reckless +++ b/tools/reckless @@ -29,6 +29,8 @@ logging.basicConfig( handlers=[logging.StreamHandler(stream=sys.stdout)], ) +LAST_FOUND = None + class Logger: """Redirect logging output to a json object or stdout as appropriate.""" @@ -52,7 +54,6 @@ class Logger: return if self.capture: self.json_output['log'].append(f"INFO: {to_log}") - self.json_output['result'].append(to_log) else: print(to_log) @@ -74,6 +75,18 @@ class Logger: else: logging.error(to_log) + def add_result(self, result: Union[str, None]): + assert json.dumps(result), "result must be json serializable" + self.json_output["result"].append(result) + + def reply_json(self): + """json output to stdout with accumulated result.""" + if len(log.json_output["result"]) == 1 and \ + isinstance(log.json_output["result"][0], list): + # unpack sources output + log.json_output["result"] = log.json_output["result"][0] + print(json.dumps(log.json_output, indent=3)) + log = Logger() @@ -1268,8 +1281,10 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: return staged_src -def install(plugin_name: str): - """downloads plugin from source repos, installs and activates plugin""" +def install(plugin_name: str) -> Union[str, None]: + """Downloads plugin from source repos, installs and activates plugin. + Returns the location of the installed plugin or "None" in the case of + failure.""" assert isinstance(plugin_name, str) # Specify a tag or commit to checkout by adding @ to plugin name if '@' in plugin_name: @@ -1279,14 +1294,16 @@ def install(plugin_name: str): name = plugin_name commit = None log.debug(f"Searching for {name}") - src = search(name) - if src: + if search(name): + global LAST_FOUND + src = LAST_FOUND src.commit = commit log.debug(f'Retrieving {src.name} from {src.source_loc}') installed = _install_plugin(src) + LAST_FOUND = None if not installed: - log.warning('installation aborted') - sys.exit(1) + log.warning(f'{plugin_name}: installation aborted') + return None # Match case of the containing directory for dirname in os.listdir(RECKLESS_CONFIG.reckless_dir): @@ -1295,14 +1312,16 @@ def install(plugin_name: str): inst_path = inst_path / dirname / installed.entry RECKLESS_CONFIG.enable_plugin(inst_path) enable(installed.name) - return + return f"{installed.source_loc}" log.error(('dynamic activation failed: ' f'{installed.name} not found in reckless directory')) - sys.exit(1) + return None + return None -def uninstall(plugin_name: str): - """disables plugin and deletes the plugin's reckless dir""" +def uninstall(plugin_name: str) -> str: + """dDisables plugin and deletes the plugin's reckless dir. Returns the + status of the uninstall attempt.""" assert isinstance(plugin_name, str) log.debug(f'Uninstalling plugin {plugin_name}') disable(plugin_name) @@ -1310,10 +1329,13 @@ def uninstall(plugin_name: str): if not Path(inst.entry).exists(): log.error("cannot find installed plugin at expected path" f"{inst.entry}") - sys.exit(1) + return "uninstall failed" log.debug(f'looking for {str(Path(inst.entry).parent)}') if remove_dir(str(Path(inst.entry).parent)): log.info(f"{inst.name} uninstalled successfully.") + else: + return "uninstall failed" + return "uninstalled" def search(plugin_name: str) -> Union[InstInfo, None]: @@ -1345,7 +1367,10 @@ def search(plugin_name: str) -> Union[InstInfo, None]: log.debug(f"entry: {found.entry}") if found.subdir: log.debug(f'sub-directory: {found.subdir}') - return found + global LAST_FOUND + # Stashing the search result saves install() a call to _source_search. + LAST_FOUND = found + return str(found.source_loc) log.debug("Search exhausted all sources") return None @@ -1409,12 +1434,14 @@ def enable(plugin_name: str): log.debug(f'{inst.name} is already running') else: log.error(f'reckless: {inst.name} failed to start!') - raise err + log.error(err) + return None except RPCError: log.debug(('lightningd rpc unavailable. ' 'Skipping dynamic activation.')) RECKLESS_CONFIG.enable_plugin(path) log.info(f'{inst.name} enabled') + return 'enabled' def disable(plugin_name: str): @@ -1425,7 +1452,7 @@ def disable(plugin_name: str): path = inst.entry if not Path(path).exists(): sys.stderr.write(f'Could not find plugin at {path}\n') - sys.exit(1) + return None log.debug(f'deactivating {plugin_name}') try: lightning_cli('plugin', 'stop', path) @@ -1434,12 +1461,14 @@ def disable(plugin_name: str): log.debug('plugin not currently running') else: log.error('lightning-cli plugin stop failed') - raise err + logging.error(err) + return None except RPCError: log.debug(('lightningd rpc unavailable. ' 'Skipping dynamic deactivation.')) RECKLESS_CONFIG.disable_plugin(path) log.info(f'{inst.name} disabled') + return 'disabled' def load_config(reckless_dir: Union[str, None] = None, @@ -1519,18 +1548,17 @@ def add_source(src: str): assert isinstance(src, str) # Is it a file? maybe_path = os.path.realpath(src) + sources = Config(path=str(get_sources_file()), + default_text='https://github.com/lightningd/plugins') if Path(maybe_path).exists(): if os.path.isdir(maybe_path): - default_repo = 'https://github.com/lightningd/plugins' - my_file = Config(path=str(get_sources_file()), - default_text=default_repo) - my_file.editConfigFile(src, None) + sources.editConfigFile(src, None) elif 'github.com' in src or 'http://' in src or 'https://' in src: - my_file = Config(path=str(get_sources_file()), - default_text='https://github.com/lightningd/plugins') - my_file.editConfigFile(src, None) + sources.editConfigFile(src, None) else: log.warning(f'failed to add source {src}') + return None + return sources_from_file() def remove_source(src: str): @@ -1543,12 +1571,20 @@ def remove_source(src: str): log.info('plugin source removed') else: log.warning(f'source not found: {src}') + return sources_from_file() def list_source(): """Provide the user with all stored source repositories.""" for src in sources_from_file(): log.info(src) + return sources_from_file() + + +def report_version() -> str: + """return reckless version""" + log.info(__VERSION__) + log.add_result(__VERSION__) class StoreIdempotent(argparse.Action): @@ -1633,6 +1669,9 @@ if __name__ == '__main__': '"reckless -h"') help_cmd.add_argument('targets', type=str, nargs='*') help_cmd.set_defaults(func=help_alias) + parser.add_argument('-V', '--version', + action=StoreTrueIdempotent, const=None, + help='print version and exit') all_parsers = [parser, install_cmd, uninstall_cmd, search_cmd, enable_cmd, disable_cmd, list_parse, source_add, source_rem, help_cmd] @@ -1655,8 +1694,6 @@ if __name__ == '__main__': type=str) p.add_argument('-v', '--verbose', action=StoreTrueIdempotent, const=None) - p.add_argument('-V', '--version', action='store_true', - help='return reckless version and exit') p.add_argument('-j', '--json', action=StoreTrueIdempotent, help='output in json format') @@ -1675,7 +1712,7 @@ if __name__ == '__main__': SUPPORTED_NETWORKS = ['bitcoin', 'regtest', 'liquid', 'liquid-regtest', 'litecoin', 'signet', 'testnet'] if args.version: - log.info(__VERSION__) + report_version() elif args.cmd1 is None: parser.print_help(sys.stdout) sys.exit(1) @@ -1719,10 +1756,9 @@ if __name__ == '__main__': args.func(args.targets) sys.exit(0) for target in args.targets: - args.func(target) + log.add_result(args.func(target)) elif 'func' in args: - args.func() + log.add_result(args.func()) - # reply with json if requested if log.capture: - print(json.dumps(log.json_output, indent=4)) + log.reply_json() From e9741e9046083e348ab684ec9b333692c72800ce Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 5 Aug 2024 16:21:05 -0500 Subject: [PATCH 10/16] reckless: don't polute stdout with python install status if --json was requested. --- tools/reckless | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tools/reckless b/tools/reckless index a11bd018e818..e9848704c205 100755 --- a/tools/reckless +++ b/tools/reckless @@ -835,7 +835,8 @@ def create_python3_venv(staged_plugin: InstInfo) -> InstInfo: # Avoid redirecting stdout in order to stream progress. # Timeout excluded as armv7 grpcio build/install can take 1hr. pip = run(['poetry', 'install', '--no-root'], check=False, - cwd=staged_plugin.source_loc, env=mod_poetry_env) + cwd=staged_plugin.source_loc, env=mod_poetry_env, + stdout=stdout_redirect, stderr=stderr_redirect) (Path(staged_plugin.source_loc) / 'pyproject.toml').unlink() (Path(staged_plugin.source_loc) / 'poetry.lock').unlink() @@ -852,7 +853,8 @@ def create_python3_venv(staged_plugin: InstInfo) -> InstInfo: elif staged_plugin.deps == 'requirements.txt': pip = run([str(env_path_full / 'bin/pip'), 'install', '-r', str(plugin_path / 'requirements.txt')], - check=False, cwd=plugin_path) + check=False, cwd=plugin_path, + stdout=stdout_redirect, stderr=stderr_redirect) else: log.debug("no python dependency file") if pip and pip.returncode != 0: @@ -911,8 +913,7 @@ def cargo_installation(cloned_plugin: InstInfo): # source_loc now contains a symlink to the entrypoint and 'source/plugin/' source = Path(cloned_plugin.source_loc) / 'source' / cloned_plugin.name log.debug(f'cargo installing from {source}') - run(['ls'], cwd=str(source), text=True, check=True) - if logging.root.level < logging.INFO: + if logging.root.level < logging.INFO and not log.capture: cargo = Popen(call, cwd=str(source), text=True) else: cargo = Popen(call, cwd=str(source), stdout=PIPE, @@ -1702,6 +1703,11 @@ if __name__ == '__main__': if args.json: log.capture = True + stdout_redirect = PIPE + stderr_redirect = PIPE + else: + stdout_redirect = None + stderr_redirect = None if args.verbose: logging.root.setLevel(logging.DEBUG) From 258514505e984dc370094b594adebd3c71ad42e4 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 30 Jul 2024 12:46:17 -0500 Subject: [PATCH 11/16] reckless: accept json array arguments as input Changelog-Added: Reckless: accepts json array input for command targets --- tools/reckless | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/reckless b/tools/reckless index e9848704c205..fe1221367502 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1588,6 +1588,18 @@ def report_version() -> str: log.add_result(__VERSION__) +def unpack_json_arg(json_target: str) -> list: + """validate json for any command line targets passes as a json array""" + try: + targets = json.loads(json_target) + except json.decoder.JSONDecodeError: + return None + if isinstance(targets, list): + return targets + log.warning(f'input {target_list} is not a json array') + return None + + class StoreIdempotent(argparse.Action): """Make the option idempotent. This adds a secondary argument that doesn't get reinitialized. The downside is it""" @@ -1762,7 +1774,13 @@ if __name__ == '__main__': args.func(args.targets) sys.exit(0) for target in args.targets: - log.add_result(args.func(target)) + # Accept single item arguments, or a json array + target_list = unpack_json_arg(target) + if target_list: + for tar in target_list: + log.add_result(args.func(tar)) + else: + log.add_result(args.func(target)) elif 'func' in args: log.add_result(args.func()) From f228e9b6414a21a39b80d6b85fd2fe7bf6d588c1 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 5 Aug 2024 18:29:44 -0500 Subject: [PATCH 12/16] reckless: handle unresolvable situations in a --json friendly way --- tools/reckless | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/tools/reckless b/tools/reckless index fe1221367502..5d453f3adf50 100755 --- a/tools/reckless +++ b/tools/reckless @@ -93,6 +93,13 @@ log = Logger() repos = ['https://github.com/lightningd/plugins'] +def reckless_abort(err: str): + log.error(err) + log.add_result(None) + log.reply_json() + sys.exit(1) + + def py_entry_guesses(name) -> list: return [name, f'{name}.py', '__init__.py'] @@ -657,7 +664,7 @@ class Config(): confirm = True sys.stdout = tmp if not confirm: - sys.exit(1) + reckless_abort(f"config file required: {config_path}") parent_path = Path(config_path).parent # Create up to one parent in the directory tree. if create_dir(parent_path): @@ -914,11 +921,10 @@ def cargo_installation(cloned_plugin: InstInfo): source = Path(cloned_plugin.source_loc) / 'source' / cloned_plugin.name log.debug(f'cargo installing from {source}') if logging.root.level < logging.INFO and not log.capture: - cargo = Popen(call, cwd=str(source), text=True) + cargo = run(call, cwd=str(source), text=True) else: - cargo = Popen(call, cwd=str(source), stdout=PIPE, - stderr=PIPE, text=True) - cargo.wait() + cargo = run(call, cwd=str(source), stdout=PIPE, + stderr=PIPE, text=True) if cargo.returncode == 0: log.debug('rust project compiled successfully') @@ -1143,7 +1149,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: log.debug(f'Install requested from {src}.') if RECKLESS_CONFIG is None: log.error('reckless install directory unavailable') - sys.exit(2) + return None # Use a unique directory for each cloned repo. tmp_path = get_temp_reckless_dir() @@ -1426,7 +1432,7 @@ def enable(plugin_name: str): path = inst.entry if not Path(path).exists(): log.error(f'cannot find installed plugin at expected path {path}') - sys.exit(1) + return None log.debug(f'activating {plugin_name}') try: lightning_cli('plugin', 'start', path) @@ -1496,10 +1502,9 @@ def load_config(reckless_dir: Union[str, None] = None, reck_conf_path = Path(reckless_dir) / f'{network}-reckless.conf' if net_conf: if str(network_path) != net_conf.conf_fp: - log.error('reckless configuration does not match lightningd:\n' - f'reckless network config path: {network_path}\n' - f'lightningd active config: {net_conf.conf_fp}') - sys.exit(1) + reckless_abort('reckless configuration does not match lightningd:\n' + f'reckless network config path: {network_path}\n' + f'lightningd active config: {net_conf.conf_fp}') else: # The network-specific config file (bitcoin by default) net_conf = LightningBitcoinConfig(path=network_path) @@ -1507,13 +1512,11 @@ def load_config(reckless_dir: Union[str, None] = None, try: reckless_conf = RecklessConfig(path=reck_conf_path) except FileNotFoundError: - log.error('reckless config file could not be written: ' - + str(reck_conf_path)) - sys.exit(1) + reckless_abort('reckless config file could not be written: ' + + str(reck_conf_path)) if not net_conf: - print('Error: could not load or create the network specific lightningd' - ' config (default .lightning/bitcoin)') - sys.exit(1) + reckless_abort('Error: could not load or create the network specific lightningd' + ' config (default .lightning/bitcoin)') net_conf.editConfigFile(f'include {reckless_conf.conf_fp}', None) return reckless_conf From b49ef8e589517fcbe9879686c4f9113c8fa81423 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 5 Aug 2024 19:09:23 -0500 Subject: [PATCH 13/16] reckless: handle other --json cases without crashing --- tools/reckless | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tools/reckless b/tools/reckless index 5d453f3adf50..8a3bea7d8182 100755 --- a/tools/reckless +++ b/tools/reckless @@ -771,6 +771,10 @@ class LightningBitcoinConfig(Config): default_text=default_text, warn=warn) +class NotFoundError(Exception): + """Raised by InferInstall when a source/entrypoint cannot be located.""" + + class InferInstall(): """Once a plugin is installed, we may need its directory and entrypoint""" def __init__(self, name: str): @@ -799,7 +803,8 @@ class InferInstall(): actual_name = reck_contents_lower[name.lower()] self.dir = Path(RECKLESS_CONFIG.reckless_dir).joinpath(actual_name) else: - raise Exception(f"Could not find a reckless directory for {name}") + raise NotFoundError("Could not find a reckless directory " + f"for {name}") plug_dir = Path(RECKLESS_CONFIG.reckless_dir).joinpath(actual_name) for guess in entry_guesses(actual_name): for content in plug_dir.iterdir(): @@ -807,7 +812,7 @@ class InferInstall(): self.entry = str(content) self.name = actual_name return - raise Exception(f'plugin entrypoint not found in {self.dir}') + raise NotFoundError(f'plugin entrypoint not found in {self.dir}') class InstallationFailure(Exception): @@ -1306,7 +1311,11 @@ def install(plugin_name: str) -> Union[str, None]: src = LAST_FOUND src.commit = commit log.debug(f'Retrieving {src.name} from {src.source_loc}') - installed = _install_plugin(src) + try: + installed = _install_plugin(src) + except FileExistsError as err: + log.error(f'File exists: {err.filename}') + return None LAST_FOUND = None if not installed: log.warning(f'{plugin_name}: installation aborted') @@ -1428,7 +1437,11 @@ def lightning_cli(*cli_args, timeout: int = 15) -> dict: def enable(plugin_name: str): """dynamically activates plugin and adds to config (persistent)""" assert isinstance(plugin_name, str) - inst = InferInstall(plugin_name) + try: + inst = InferInstall(plugin_name) + except NotFoundError as err: + log.error(err) + return None path = inst.entry if not Path(path).exists(): log.error(f'cannot find installed plugin at expected path {path}') @@ -1439,6 +1452,7 @@ def enable(plugin_name: str): except CLIError as err: if 'already registered' in err.message: log.debug(f'{inst.name} is already running') + return None else: log.error(f'reckless: {inst.name} failed to start!') log.error(err) @@ -1455,7 +1469,11 @@ def disable(plugin_name: str): """reckless disable deactivates an installed plugin""" assert isinstance(plugin_name, str) - inst = InferInstall(plugin_name) + try: + inst = InferInstall(plugin_name) + except NotFoundError as err: + log.warning(f'failed to disable: {err}') + return None path = inst.entry if not Path(path).exists(): sys.stderr.write(f'Could not find plugin at {path}\n') From 2e127a7c91c7c8ab84fc14c1ff7cc344cc50517f Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 6 Aug 2024 07:54:28 -0500 Subject: [PATCH 14/16] reckless: handle failure to find entrypoint cleanly --- tools/reckless | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/reckless b/tools/reckless index 8a3bea7d8182..a9409e39f02c 100755 --- a/tools/reckless +++ b/tools/reckless @@ -1341,7 +1341,11 @@ def uninstall(plugin_name: str) -> str: assert isinstance(plugin_name, str) log.debug(f'Uninstalling plugin {plugin_name}') disable(plugin_name) - inst = InferInstall(plugin_name) + try: + inst = InferInstall(plugin_name) + except NotFoundError as err: + log.error(err) + return "uninstall failed" if not Path(inst.entry).exists(): log.error("cannot find installed plugin at expected path" f"{inst.entry}") From 859cf03f9026bf324c6908f9d04753e67e701e4f Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 6 Aug 2024 12:13:29 -0500 Subject: [PATCH 15/16] reckless: escape strings in log --- tools/reckless | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/reckless b/tools/reckless index a9409e39f02c..1ab0d7bd0b4e 100755 --- a/tools/reckless +++ b/tools/reckless @@ -39,12 +39,16 @@ class Logger: "log": []} self.capture = capture + def str_esc(self, raw_string: str) -> str: + assert isinstance(raw_string, str) + return json.dumps(raw_string)[1:-1] + def debug(self, to_log: str): assert isinstance(to_log, str) or hasattr(to_log, "__repr__") if logging.root.level > logging.DEBUG: return if self.capture: - self.json_output['log'].append(f"DEBUG: {to_log}") + self.json_output['log'].append(self.str_esc(f"DEBUG: {to_log}")) else: logging.debug(to_log) @@ -53,7 +57,7 @@ class Logger: if logging.root.level > logging.INFO: return if self.capture: - self.json_output['log'].append(f"INFO: {to_log}") + self.json_output['log'].append(self.str_esc(f"INFO: {to_log}")) else: print(to_log) @@ -62,7 +66,7 @@ class Logger: if logging.root.level > logging.WARNING: return if self.capture: - self.json_output['log'].append(f"WARNING: {to_log}") + self.json_output['log'].append(self.str_esc(f"WARNING: {to_log}")) else: logging.warning(to_log) @@ -71,7 +75,7 @@ class Logger: if logging.root.level > logging.ERROR: return if self.capture: - self.json_output['log'].append(f"ERROR: {to_log}") + self.json_output['log'].append(self.str_esc(f"ERROR: {to_log}")) else: logging.error(to_log) From 87895bd99f25bcc72d2c3db58fd7829a166290ad Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 7 Aug 2024 09:54:28 -0500 Subject: [PATCH 16/16] reckless: correct logging levels Some output was hidden under normal operation which should not have been. --- tools/reckless | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/reckless b/tools/reckless index 1ab0d7bd0b4e..f029daafd53b 100755 --- a/tools/reckless +++ b/tools/reckless @@ -677,8 +677,8 @@ class Config(): # FIXME: Handle write failure return default_text else: - log.debug('could not create the parent directory ' + - parent_path) + log.warning('could not create the parent directory ' + + parent_path) raise FileNotFoundError('invalid parent directory') def editConfigFile(self, addline: Union[str, None], @@ -874,7 +874,6 @@ def create_python3_venv(staged_plugin: InstInfo) -> InstInfo: else: log.debug("no python dependency file") if pip and pip.returncode != 0: - log.debug("install to virtual environment failed") log.error('error encountered installing dependencies') raise InstallationFailure @@ -1191,7 +1190,7 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: cloned_src = _source_search(src.name, str(clone_path)) log.debug(f'cloned_src: {cloned_src}') if not cloned_src: - log.debug('failed to find plugin after cloning repo.') + log.warning('failed to find plugin after cloning repo.') return None # If a specific commit or tag was requested, check it out now. @@ -1211,7 +1210,8 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]: INSTALLER = inst_method break if not INSTALLER: - log.debug('Could not find a suitable installer method.') + log.warning('Could not find a suitable installer method for ' + f'{src.name}') return None if not cloned_src.entry: # The plugin entrypoint may not be discernable prior to cloning. @@ -1395,7 +1395,7 @@ def search(plugin_name: str) -> Union[InstInfo, None]: # Stashing the search result saves install() a call to _source_search. LAST_FOUND = found return str(found.source_loc) - log.debug("Search exhausted all sources") + log.info("Search exhausted all sources") return None