From d5efecbd675116ec29ca5bfc9d590044458b6d97 Mon Sep 17 00:00:00 2001 From: Marco Esters Date: Tue, 1 Oct 2024 10:56:34 -0700 Subject: [PATCH] Improve standalone executable type detection and handling (#864) * Replace string values for standalone binary with enum * Explicitly test for micromamba * Replace is_micromamba with enum comparison * Remove circular import * Convert Path objects into str in indentify_conda_exe * Add news file * Add guard for None-type exe_version * Add missing guard for None-type exe versions * Loosen requirements for mamba help text * Add guards for None-type versions for skips * Do not use Enum * Update constructor/utils.py Co-authored-by: jaimergp * Simplify logger warning logic * Skip comments in explicit env file tests --------- Co-authored-by: jaimergp --- constructor/main.py | 26 ++++++----- constructor/utils.py | 46 ++++++++++++-------- constructor/winexe.py | 5 ++- news/864-improve-standalone-binary-detection | 19 ++++++++ tests/test_examples.py | 27 +++++++++--- 5 files changed, 86 insertions(+), 37 deletions(-) create mode 100644 news/864-improve-standalone-binary-detection diff --git a/constructor/main.py b/constructor/main.py index 98e6a080c..2f724bafd 100644 --- a/constructor/main.py +++ b/constructor/main.py @@ -21,7 +21,7 @@ from .construct import parse as construct_parse from .construct import verify as construct_verify from .fcp import main as fcp_main -from .utils import identify_conda_exe, normalize_path, yield_lines +from .utils import StandaloneExe, identify_conda_exe, normalize_path, yield_lines DEFAULT_CACHE_DIR = os.getenv('CONSTRUCTOR_CACHE', '~/.conda/constructor') @@ -92,7 +92,13 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, if platform != cc_platform and 'pkg' in itypes and not cc_platform.startswith('osx-'): sys.exit("Error: cannot construct a macOS 'pkg' installer on '%s'" % cc_platform) - if osname == "win" and "micromamba" in os.path.basename(info['_conda_exe']): + + exe_type, exe_version = identify_conda_exe(info.get("_conda_exe")) + if exe_version is not None: + exe_version = Version(exe_version) + info["_conda_exe_type"] = exe_type + info["_conda_exe_version"] = exe_version + if osname == "win" and exe_type == StandaloneExe.MAMBA: # TODO: Investigate errors on Windows and re-enable sys.exit("Error: micromamba is not supported on Windows installers.") @@ -172,17 +178,13 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, else: env_config[config_key] = value - try: - exe_name, exe_version = identify_conda_exe(info.get("_conda_exe")) - except OSError as exc: + if exe_type is None or exe_version is None: logger.warning( - "Could not identify conda-standalone / micromamba version (%s). " - "Will assume it is compatible with shortcuts.", - exc, - ) - exe_name, exe_version = None, None - if sys.platform != "win32" and exe_name is not None and ( - exe_name == "micromamba" or Version(exe_version) < Version("23.11.0") + "Could not identify conda-standalone / micromamba version. " + "Will assume it is compatible with shortcuts." + ) + elif sys.platform != "win32" and ( + exe_type != StandaloneExe.CONDA or exe_version < Version("23.11.0") ): logger.warning("conda-standalone 23.11.0 or above is required for shortcuts on Unix.") info['_enable_shortcuts'] = "incompatible" diff --git a/constructor/utils.py b/constructor/utils.py index 7bdf1ec5c..dc44f838e 100644 --- a/constructor/utils.py +++ b/constructor/utils.py @@ -11,9 +11,11 @@ import sys from io import StringIO from os import environ, sep, unlink -from os.path import basename, isdir, isfile, islink, join, normpath +from os.path import isdir, isfile, islink, join, normpath +from pathlib import Path from shutil import rmtree -from subprocess import check_call, check_output +from subprocess import CalledProcessError, check_call, check_output +from typing import Tuple, Union from ruamel.yaml import YAML @@ -23,6 +25,11 @@ yaml.indent(mapping=2, sequence=4, offset=2) +class StandaloneExe: + CONDA = "conda" + MAMBA = "mamba" + + def explained_check_call(args): """ Execute a system process and debug the invocation @@ -153,7 +160,7 @@ def ensure_transmuted_ext(info, url): """ if ( info.get("transmute_file_type") == ".conda" - and "micromamba" in basename(info.get("_conda_exe", "")) + and info.get("_conda_exe_type") == StandaloneExe.MAMBA ): if url.lower().endswith(".tar.bz2"): url = url[:-8] + ".conda" @@ -215,15 +222,13 @@ def yield_lines(path): yield line -def shortcuts_flags(info, conda_exe=None): +def shortcuts_flags(info) -> str: menu_packages = info.get("menu_packages") - conda_exe = conda_exe or info.get("_conda_exe", "") - is_micromamba = "micromamba" in basename(conda_exe).lower() if menu_packages is None: # not set: we create all shortcuts (default behaviour) return "" if menu_packages: - if is_micromamba: + if info.get("_conda_exe_type") == StandaloneExe.MAMBA: logger.warning( "Micromamba does not support '--shortcuts-only'. " "Will install all shortcuts." @@ -252,19 +257,24 @@ def approx_size_kb(info, which="pkgs"): return int(math.ceil(size_bytes/1000)) -def identify_conda_exe(conda_exe=None): +def identify_conda_exe(conda_exe: Union[str, Path] = None) -> Tuple[StandaloneExe, str]: if conda_exe is None: conda_exe = normalize_path(join(sys.prefix, "standalone_conda", "conda.exe")) - output = check_output([conda_exe, "--version"], text=True) - output = output.strip() - fields = output.split() - if "conda" in fields: - name = "conda-standalone" - version = fields[1] - else: - name = "micromamba" - version = output.strip() - return name, version + if isinstance(conda_exe, Path): + conda_exe = str(conda_exe) + try: + output_version = check_output([conda_exe, "--version"], text=True) + output_version = output_version.strip() + fields = output_version.split() + if "conda" in fields: + return StandaloneExe.CONDA, fields[1] + # micromamba only returns the version number + output_help = check_output([conda_exe, "--help"], text=True) + if "mamba" in output_help: + return StandaloneExe.MAMBA, output_version + except CalledProcessError as exc: + logger.warning(f"Could not identify standalone binary {exc}.") + return None, None def win_str_esc(s, newlines=True): diff --git a/constructor/winexe.py b/constructor/winexe.py index 8d6d7fd4c..16bcca7b5 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -172,6 +172,9 @@ def setup_envs_commands(info, dir_path): for env_name in info.get("_extra_envs_info", {}): lines += ["", ""] env_info = info["extra_envs"][env_name] + # Needed for shortcuts_flags function + if "_conda_exe_type" not in env_info: + env_info["_conda_exe_type"] = info.get("_conda_exe_type") channel_info = { "channels": env_info.get("channels", info.get("channels", ())), "channels_remap": env_info.get("channels_remap", info.get("channels_remap", ())) @@ -185,7 +188,7 @@ def setup_envs_commands(info, dir_path): conda_meta=join("$INSTDIR", "envs", env_name, "conda-meta"), history_abspath=join(dir_path, "envs", env_name, "conda-meta", "history"), channels=",".join(get_final_channels(channel_info)), - shortcuts=shortcuts_flags(env_info, conda_exe=info.get("_conda_exe")), + shortcuts=shortcuts_flags(env_info), register_envs=str(info.get("register_envs", True)).lower(), ).splitlines() diff --git a/news/864-improve-standalone-binary-detection b/news/864-improve-standalone-binary-detection new file mode 100644 index 000000000..761b0a3fa --- /dev/null +++ b/news/864-improve-standalone-binary-detection @@ -0,0 +1,19 @@ +### Enhancements + +* Improve detection and handling of standalone executable type. (#864) + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/test_examples.py b/tests/test_examples.py index 380d68f6f..549cfd835 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -18,7 +18,7 @@ from conda.core.prefix_data import PrefixData from conda.models.version import VersionOrder as Version -from constructor.utils import identify_conda_exe +from constructor.utils import StandaloneExe, identify_conda_exe if sys.platform == "darwin": from constructor.osxpkg import calculate_install_dir @@ -36,6 +36,8 @@ ON_CI = os.environ.get("CI") CONSTRUCTOR_CONDA_EXE = os.environ.get("CONSTRUCTOR_CONDA_EXE") CONDA_EXE, CONDA_EXE_VERSION = identify_conda_exe(CONSTRUCTOR_CONDA_EXE) +if CONDA_EXE_VERSION is not None: + CONDA_EXE_VERSION = Version(CONDA_EXE_VERSION) CONSTRUCTOR_DEBUG = bool(os.environ.get("CONSTRUCTOR_DEBUG")) if artifacts_path := os.environ.get("CONSTRUCTOR_EXAMPLES_KEEP_ARTIFACTS"): KEEP_ARTIFACTS_PATH = Path(artifacts_path) @@ -355,8 +357,9 @@ def _example_path(example_name): return REPO_DIR / "examples" / example_name -def _is_micromamba(path): - return "micromamba" in Path(path).stem +def _is_micromamba(path) -> bool: + name, _ = identify_conda_exe(path) + return name == StandaloneExe.MAMBA def test_example_customize_controls(tmp_path, request): @@ -391,7 +394,11 @@ def test_example_extra_files(tmp_path, request): @pytest.mark.xfail( - CONDA_EXE == "conda-standalone" and Version(CONDA_EXE_VERSION) < Version("23.11.0a0"), + ( + CONDA_EXE == StandaloneExe.CONDA + and CONDA_EXE_VERSION is not None + and CONDA_EXE_VERSION < Version("23.11.0a0") + ), reason="Known issue with conda-standalone<=23.10: shortcuts are created but not removed.", ) def test_example_miniforge(tmp_path, request): @@ -565,7 +572,11 @@ def test_example_scripts(tmp_path, request): @pytest.mark.skipif( - CONDA_EXE == "micromamba" or Version(CONDA_EXE_VERSION) < Version("23.11.0a0"), + ( + CONDA_EXE == StandaloneExe.MAMBA + or CONDA_EXE_VERSION is None + or CONDA_EXE_VERSION < Version("23.11.0a0") + ), reason="menuinst v2 requires conda-standalone>=23.11.0; micromamba is not supported yet", ) def test_example_shortcuts(tmp_path, request): @@ -723,7 +734,11 @@ def test_example_from_explicit(tmp_path, request): [sys.executable, "-mconda", "list", "-p", install_dir, "--explicit", "--md5"], text=True, ) - assert out == (input_path / "explicit_linux-64.txt").read_text() + expected = (input_path / "explicit_linux-64.txt").read_text() + # Filter comments + out = [line for line in out.split("\n") if not line.startswith("#")] + expected = [line for line in expected.split("\n") if not line.startswith("#")] + assert out == expected def test_register_envs(tmp_path, request):