From e61aef21cdaaac04a227acbdfefb3fabd4f9bba5 Mon Sep 17 00:00:00 2001 From: Marco Esters Date: Fri, 8 Nov 2024 07:20:38 -0800 Subject: [PATCH 1/3] Ignore .condarc files during installation (#863) * Create test for ignoring .condarc files * Set CONDA_RESTRICT_RC_SEARCH_PATH for all installers * Add --no-rc option to micromamba and conda-standalone calls * Simplify micromamba test logic * Monkeypatch XDG_CONFIG_HOME for conda-standalone test * Add news * Use different installer name to avoid conflicting artifacts * Use StanadloneExe for binary type checking * Use new exe type in tests --------- Co-authored-by: jaimergp --- constructor/header.sh | 6 ++-- constructor/main.py | 9 +++++ constructor/nsis/main.nsi.tmpl | 4 +-- constructor/osx/prepare_installation.sh | 2 +- constructor/osx/run_installation.sh | 4 +-- constructor/osxpkg.py | 1 + constructor/shar.py | 3 +- constructor/winexe.py | 7 ++-- news/863-ignore-condarc-files | 19 +++++++++++ tests/test_examples.py | 44 +++++++++++++++++++++++++ 10 files changed, 88 insertions(+), 11 deletions(-) create mode 100644 news/863-ignore-condarc-files diff --git a/constructor/header.sh b/constructor/header.sh index b1d38be9d..f07212f95 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -483,7 +483,7 @@ if [ "__VIRTUAL_SPECS__" != "" ]; then CONDA_QUIET="$BATCH" \ CONDA_SOLVER="classic" \ CONDA_PKGS_DIRS="$(mktemp -d)" \ - "$CONDA_EXEC" create --dry-run --prefix "$PREFIX/envs/_virtual_specs_checks" --offline __VIRTUAL_SPECS__ + "$CONDA_EXEC" create --dry-run --prefix "$PREFIX/envs/_virtual_specs_checks" --offline __VIRTUAL_SPECS__ __NO_RCS_ARG__ fi # Create $PREFIX/.nonadmin if the installation didn't require superuser permissions @@ -561,7 +561,7 @@ CONDA_EXTRA_SAFETY_CHECKS=no \ CONDA_CHANNELS="__CHANNELS__" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ CONDA_QUIET="$BATCH" \ -"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts || exit 1 +"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts __NO_RCS_ARG__ || exit 1 rm -f "$PREFIX/pkgs/env.txt" #The templating doesn't support nested if statements @@ -606,7 +606,7 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do CONDA_CHANNELS="$env_channels" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ CONDA_QUIET="$BATCH" \ - "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts || exit 1 + "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts __NO_RCS_ARG__ || exit 1 rm -f "${env_pkgs}env.txt" done #endif diff --git a/constructor/main.py b/constructor/main.py index 2f724bafd..5f8a95429 100644 --- a/constructor/main.py +++ b/constructor/main.py @@ -199,6 +199,15 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, ) ) + # Add --no-rc option to CONDA_EXE command so that existing + # .condarc files do not pollute the installation process. + if exe_type == StandaloneExe.CONDA and exe_version and exe_version >= Version("24.9.0"): + info["_ignore_condarcs_arg"] = "--no-rc" + elif exe_type == StandaloneExe.MAMBA: + info["_ignore_condarcs_arg"] = "--no-rc" + else: + info["_ignore_condarcs_arg"] = "" + if 'pkg' in itypes: if (domains := info.get('pkg_domains')) is not None: domains = {key: str(val).lower() for key, val in domains.items()} diff --git a/constructor/nsis/main.nsi.tmpl b/constructor/nsis/main.nsi.tmpl index 125901856..3bb66380e 100644 --- a/constructor/nsis/main.nsi.tmpl +++ b/constructor/nsis/main.nsi.tmpl @@ -1263,7 +1263,7 @@ Section "Install" System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_SOLVER", "classic").r0' SetDetailsPrint TextOnly ${Print} "Checking virtual specs compatibility: @VIRTUAL_SPECS_DEBUG@" - push '"$INSTDIR\_conda.exe" create --dry-run --prefix "$INSTDIR\envs\_virtual_specs_checks" --offline @VIRTUAL_SPECS@' + push '"$INSTDIR\_conda.exe" create --dry-run --prefix "$INSTDIR\envs\_virtual_specs_checks" --offline @VIRTUAL_SPECS@ @NO_RCS_ARG@' push 'Failed to check virtual specs: @VIRTUAL_SPECS_DEBUG@' push 'WithLog' call AbortRetryNSExecWait @@ -1325,7 +1325,7 @@ Section "Install" ${If} $Ana_ClearPkgCache_State = ${BST_CHECKED} ${Print} "Clearing package cache..." - push '"$INSTDIR\_conda.exe" clean --all --force-pkgs-dirs --yes' + push '"$INSTDIR\_conda.exe" clean --all --force-pkgs-dirs --yes @NO_RCS_ARG@' push 'Failed to clear package cache' push 'WithLog' call AbortRetryNSExecWait diff --git a/constructor/osx/prepare_installation.sh b/constructor/osx/prepare_installation.sh index 35f7b3e17..17f13e4b4 100644 --- a/constructor/osx/prepare_installation.sh +++ b/constructor/osx/prepare_installation.sh @@ -42,7 +42,7 @@ if [ "__VIRTUAL_SPECS__" != "" ]; then notify 'Checking virtual specs compatibility: __VIRTUAL_SPECS__' CONDA_SOLVER="classic" \ CONDA_PKGS_DIRS="$(mktemp -d)" \ - "$CONDA_EXEC" create --dry-run --prefix "$PREFIX/envs/_virtual_specs_checks" --offline __VIRTUAL_SPECS__ + "$CONDA_EXEC" create --dry-run --prefix "$PREFIX/envs/_virtual_specs_checks" --offline __VIRTUAL_SPECS__ __NO_RCS_ARG__ fi # Create $PREFIX/.nonadmin if the installation didn't require superuser permissions diff --git a/constructor/osx/run_installation.sh b/constructor/osx/run_installation.sh index 9258c12fc..ba5aa90d0 100644 --- a/constructor/osx/run_installation.sh +++ b/constructor/osx/run_installation.sh @@ -48,7 +48,7 @@ CONDA_SAFETY_CHECKS=disabled \ CONDA_EXTRA_SAFETY_CHECKS=no \ CONDA_CHANNELS=__CHANNELS__ \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ -"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts; then +"$CONDA_EXEC" install --offline --file "$PREFIX/pkgs/env.txt" -yp "$PREFIX" $shortcuts __NO_RCS_ARG__; then echo "ERROR: could not complete the conda install" exit 1 fi @@ -94,7 +94,7 @@ for env_pkgs in "${PREFIX}"/pkgs/envs/*/; do CONDA_EXTRA_SAFETY_CHECKS=no \ CONDA_CHANNELS="$env_channels" \ CONDA_PKGS_DIRS="$PREFIX/pkgs" \ - "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts || exit 1 + "$CONDA_EXEC" install --offline --file "${env_pkgs}env.txt" -yp "$PREFIX/envs/$env_name" $env_shortcuts __NO_RCS_ARG__ || exit 1 # Move the prepackaged history file into place mv "${env_pkgs}/conda-meta/history" "$PREFIX/envs/$env_name/conda-meta/history" rm -f "${env_pkgs}env.txt" diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index a2280adf2..6c7c1e6c5 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -345,6 +345,7 @@ def move_script(src, dst, info, ensure_shebang=False, user_script_type=None): 'ENABLE_SHORTCUTS': str(info['_enable_shortcuts']).lower(), 'REGISTER_ENVS': str(info.get("register_envs", True)).lower(), 'VIRTUAL_SPECS': shlex.join(info.get("virtual_specs", ())), + 'NO_RCS_ARG': info.get('_ignore_condarcs_arg', ''), } data = preprocess(data, ppd) custom_variables = info.get('script_env_variables', {}) diff --git a/constructor/shar.py b/constructor/shar.py index 7e5a5157e..768e5a4b1 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -94,7 +94,8 @@ def get_header(conda_exec, tarball, info): 'SHORTCUTS': shortcuts_flags(info), 'REGISTER_ENVS': str(info.get("register_envs", True)).lower(), 'TOTAL_INSTALLATION_SIZE_KB': str(approx_size_kb(info, "total")), - 'VIRTUAL_SPECS': shlex.join(info.get("virtual_specs", ())) + 'VIRTUAL_SPECS': shlex.join(info.get("virtual_specs", ())), + 'NO_RCS_ARG': info.get('_ignore_condarcs_arg', ''), } if has_license: replace['LICENSE'] = read_ascii_only(info['license_file']) diff --git a/constructor/winexe.py b/constructor/winexe.py index 16bcca7b5..cc9840340 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -135,10 +135,10 @@ def setup_envs_commands(info, dir_path): # Run conda install ${{If}} $Ana_CreateShortcuts_State = ${{BST_CHECKED}} ${{Print}} "Installing packages for {name}, creating shortcuts if necessary..." - push '"$INSTDIR\_conda.exe" install --offline -yp "{prefix}" --file "{env_txt}" {shortcuts}' + push '"$INSTDIR\_conda.exe" install --offline -yp "{prefix}" --file "{env_txt}" {shortcuts} {no_rcs_arg}' ${{Else}} ${{Print}} "Installing packages for {name}..." - push '"$INSTDIR\_conda.exe" install --offline -yp "{prefix}" --file "{env_txt}" --no-shortcuts' + push '"$INSTDIR\_conda.exe" install --offline -yp "{prefix}" --file "{env_txt}" --no-shortcuts {no_rcs_arg}' ${{EndIf}} push 'Failed to link extracted packages to {prefix}!' push 'WithLog' @@ -167,6 +167,7 @@ def setup_envs_commands(info, dir_path): channels=','.join(get_final_channels(info)), shortcuts=shortcuts_flags(info), register_envs=str(info.get("register_envs", True)).lower(), + no_rcs_arg=info.get("_ignore_condarcs_arg", ""), ).splitlines() # now we generate one more block per extra env, if present for env_name in info.get("_extra_envs_info", {}): @@ -190,6 +191,7 @@ def setup_envs_commands(info, dir_path): channels=",".join(get_final_channels(channel_info)), shortcuts=shortcuts_flags(env_info), register_envs=str(info.get("register_envs", True)).lower(), + no_rcs_arg=info.get("_ignore_condarcs_arg", ""), ).splitlines() return [line.strip() for line in lines] @@ -404,6 +406,7 @@ def make_nsi( # This is the same but without quotes so we can print it fine ('@VIRTUAL_SPECS_DEBUG@', " ".join([spec for spec in info.get("virtual_specs", ())])), ('@LICENSEFILENAME@', basename(info.get('license_file', 'placeholder_license.txt'))), + ('@NO_RCS_ARG@', info.get('_ignore_condarcs_arg', '')), ]: data = data.replace(key, value) diff --git a/news/863-ignore-condarc-files b/news/863-ignore-condarc-files new file mode 100644 index 000000000..653fa4a49 --- /dev/null +++ b/news/863-ignore-condarc-files @@ -0,0 +1,19 @@ +### Enhancements + +* + +### Bug fixes + +* Ignore pre-existing .condarc files to prevent these configuration files from interfering with the installation process. (#542 and #568 via #863) + +### Deprecations + +* + +### Docs + +* + +### Other + +* diff --git a/tests/test_examples.py b/tests/test_examples.py index 549cfd835..3e4405f1f 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -842,3 +842,47 @@ def test_virtual_specs_ok(tmp_path, request): check_subprocess=True, uninstall=True, ) + + +@pytest.mark.xfail( + CONDA_EXE == StandaloneExe.CONDA and CONDA_EXE_VERSION < Version("24.9.0"), + reason="Pre-existing .condarc breaks installation", +) +def test_ignore_condarc_files(tmp_path, monkeypatch, request): + # Create a bogus .condarc file that would result in errors if read. + # conda searches inside XDG_CONFIG_HOME on all systems, which is a + # a safer directory to monkeypatch, especially on Windows where patching + # HOME or USERPROFILE breaks installer builds. + # mamba does not search this directory, so use HOME as a fallback. + # Since micromamba is not supported on Windows, this is not a problem. + if CONDA_EXE == StandaloneExe.MAMBA: + monkeypatch.setenv("HOME", str(tmp_path)) + condarc = tmp_path / ".condarc" + else: + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + condarc = tmp_path / "conda" / ".condarc" + condarc.parent.mkdir(parents=True, exist_ok=True) + condarc.write_text("safety_checks:\n - very safe\n") + recipe_path = _example_path("customize_controls") + input_path = tmp_path / "input" + shutil.copytree(str(recipe_path), str(input_path)) + # Rewrite installer name to avoid duplicate artifacts + construct_yaml = input_path / "construct.yaml" + content = construct_yaml.read_text() + construct_yaml.write_text(content.replace("name: NoCondaOptions", "name: NoCondaRC")) + for installer, install_dir in create_installer(input_path, tmp_path): + proc = _run_installer( + input_path, + installer, + install_dir, + request=request, + check_subprocess=True, + uninstall=True, + ) + if CONDA_EXE == StandaloneExe.MAMBA and installer.suffix == ".sh": + # micromamba loads the rc files even for constructor subcommands. + # This cannot be turned off with --no-rc, which causes four errors + # in stderr. If there are more, other micromamba calls have read + # the bogus .condarc file. + # pkg installers unfortunately do not output any errors into the log. + assert proc.stderr.count("Bad conversion of configurable") == 4 From 6c430859e1d24ecafddb2d4acefc58db26fc89a4 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 8 Nov 2024 17:58:57 +0100 Subject: [PATCH 2/3] Ensure `--output-dir` always exists (#891) --- constructor/main.py | 1 + news/891-output-dir | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 news/891-output-dir diff --git a/constructor/main.py b/constructor/main.py index 5f8a95429..79fa8c3ef 100644 --- a/constructor/main.py +++ b/constructor/main.py @@ -250,6 +250,7 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, # '_dists': List[Dist] # '_urls': List[Tuple[url, md5]] + os.makedirs(output_dir, exist_ok=True) info_dicts = [] for itype in itypes: if itype == 'sh': diff --git a/news/891-output-dir b/news/891-output-dir new file mode 100644 index 000000000..9fcc5adc2 --- /dev/null +++ b/news/891-output-dir @@ -0,0 +1,19 @@ +### Enhancements + +* + +### Bug fixes + +* Do not crash if `--output-dir` doesn't exist. Ensure it does before creating installers. (#772 via #891) + +### Deprecations + +* + +### Docs + +* + +### Other + +* From 6b551ffb2bb27a896751843f94df9290515c7b80 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 8 Nov 2024 17:59:07 +0100 Subject: [PATCH 3/3] Always print NSIS output on errors (#890) --- constructor/winexe.py | 10 +++++++--- news/890-debug-nsis | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 news/890-debug-nsis diff --git a/constructor/winexe.py b/constructor/winexe.py index cc9840340..7f8c9968d 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -499,9 +499,13 @@ def create(info, verbose=False): args = [MAKENSIS_EXE, verbosity, nsi] logger.info('Calling: %s', args) process = run(args, capture_output=True, text=True) - logger.debug("makensis stdout:\n'%s'", process.stdout) - logger.debug("makensis stderr:\n'%s'", process.stderr) - process.check_returncode() + if process.returncode: + logger.info("makensis stdout:\n'%s'", process.stdout) + logger.error("makensis stderr:\n'%s'", process.stderr) + sys.exit(f"Failed to run {args}. Exit code: {process.returncode}.") + else: + logger.debug("makensis stdout:\n'%s'", process.stdout) + logger.debug("makensis stderr:\n'%s'", process.stderr) if signing_tool: signing_tool.verify_signature(info['_outpath']) diff --git a/news/890-debug-nsis b/news/890-debug-nsis new file mode 100644 index 000000000..2daa2a9d2 --- /dev/null +++ b/news/890-debug-nsis @@ -0,0 +1,19 @@ +### Enhancements + +* + +### Bug fixes + +* Always report NSIS output when an error occurs. (#798 via #890) + +### Deprecations + +* + +### Docs + +* + +### Other + +*