From 46c8f19beed9dfa19bf1845a66a32e1d4f5090fe Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 11:07:04 +0200 Subject: [PATCH 01/34] Add support for virtual_specs checks before installation --- constructor/construct.py | 5 +++++ constructor/header.sh | 7 +++++++ constructor/main.py | 4 ++-- constructor/nsis/main.nsi.tmpl | 12 ++++++++++++ constructor/osxpkg.py | 21 ++++++++++++++++++++- constructor/shar.py | 2 ++ constructor/winexe.py | 1 + examples/virtual_specs/construct.yaml | 20 ++++++++++++++++++++ tests/test_examples.py | 8 ++++++++ 9 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 examples/virtual_specs/construct.yaml diff --git a/constructor/construct.py b/constructor/construct.py index 2fe4c8978..e5c33b4fd 100644 --- a/constructor/construct.py +++ b/constructor/construct.py @@ -81,6 +81,11 @@ for example, if `python=3.6` is included, then conda will always seek versions of packages compatible with Python 3.6. If this is option is not provided, it will be set equal to the value of `specs`. +'''), + + ('virtual_specs', False, list, ''' +A list of virtual packages that must be satisfied at install time.Virtual +packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. '''), ('exclude', False, list, ''' diff --git a/constructor/header.sh b/constructor/header.sh index 0fdbee3b5..da6132c43 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -423,6 +423,13 @@ export TMP_BACKUP="${TMP:-}" export TMP="$PREFIX/install_tmp" mkdir -p "$TMP" +# Check whether the virtual specs can be satisfied +if [ "__VIRTUAL_SPECS__" != "" ]; then + CONDA_QUIET="$BATCH" \ + CONDA_SOLVER="classic" \ + "$CONDA_EXEC" create --dry-run --prefix "$PREFIX" --offline __VIRTUAL_SPECS__ +fi + # Create $PREFIX/.nonadmin if the installation didn't require superuser permissions if [ "$(id -u)" -ne 0 ]; then touch "$PREFIX/.nonadmin" diff --git a/constructor/main.py b/constructor/main.py index c2223a6bf..346882a95 100644 --- a/constructor/main.py +++ b/constructor/main.py @@ -115,7 +115,7 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, elif info.get("signing_certificate"): info["windows_signing_tool"] = "signtool" - for key in 'specs', 'packages': + for key in 'specs', 'packages', 'virtual_specs': if key not in info: continue if isinstance(info[key], str): @@ -137,7 +137,7 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, new_extras.append({orig: dest}) info[extra_type] = new_extras - for key in 'channels', 'specs', 'exclude', 'packages', 'menu_packages': + for key in 'channels', 'specs', 'exclude', 'packages', 'menu_packages', 'virtual_specs': if key in info: # ensure strings in those lists are stripped info[key] = [line.strip() for line in info[key]] diff --git a/constructor/nsis/main.nsi.tmpl b/constructor/nsis/main.nsi.tmpl index 664e79bdc..28332462a 100644 --- a/constructor/nsis/main.nsi.tmpl +++ b/constructor/nsis/main.nsi.tmpl @@ -1132,6 +1132,18 @@ Section "Install" System::Call 'kernel32::SetEnvironmentVariable(t,t)i("INSTALLER_PLAT", "${PLATFORM}").r0' System::Call 'kernel32::SetEnvironmentVariable(t,t)i("INSTALLER_TYPE", "EXE").r0' + ${If} '__VIRTUAL_SPECS__' != '' + System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_SOLVER", "classic").r0' + SetDetailsPrint TextOnly + DetailPrint "Checking virtual specs..." + push '"$INSTDIR\_conda.exe" create --dry-run --prefix "$INSTDIR" --offline __VIRTUAL_SPECS__' + push 'Virtual specs checks failed' + push 'WithLog' + call AbortRetryNSExecWait + SetDetailsPrint both + System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_SOLVER", "").r0' + ${EndIf} + @PKG_COMMANDS@ SetDetailsPrint TextOnly diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index ca55934ca..d0951d5eb 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -9,7 +9,7 @@ from tempfile import NamedTemporaryFile from . import preconda -from .conda_interface import conda_context +from .conda_interface import conda_context, MatchSpec from .construct import ns_platform, parse from .imaging import write_images from .utils import ( @@ -188,6 +188,25 @@ def modify_xml(xml_path, info): ) root.append(readme) + # -- __osx virtual package checks -- # + osx_versions = {} + for spec in info.get("virtual_specs", ()): + spec = MatchSpec(spec) + if spec.name != "__osx": + continue + if not spec.version: + continue + operator = spec.version.operator_func.__name__ + if operator == "ge": + osx_versions["min"] = str(spec.version.matcher_vo) + elif operator == "le": + osx_versions["max"] = str(spec.version.matcher_vo) + + if osx_versions: + allowed_os_versions = ET.Element("allowed-os-versions") + allowed_os_versions.append(ET.Element("os-version", osx_versions)) + root.append(allowed_os_versions) + # See below for an explanation of the consequences of this # customLocation value. for options in root.findall('options'): diff --git a/constructor/shar.py b/constructor/shar.py index 9f2d90425..9941967bc 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -6,6 +6,7 @@ import logging import os +import shlex import shutil import stat import tarfile @@ -92,6 +93,7 @@ def get_header(conda_exec, tarball, info): 'SHORTCUTS': shortcuts_flags(info), 'REGISTER_ENVS': str(info.get("register_envs", True)).lower(), 'TOTAL_INSTALLATION_SIZE_KB': str(approx_size_kb(info, "total")), + 'VIRTUAL_SPECS': shlex.join(info.get("virtual_specs", ())) } if has_license: replace['LICENSE'] = read_ascii_only(info['license_file']) diff --git a/constructor/winexe.py b/constructor/winexe.py index 3afbd8f9a..9412c4223 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -274,6 +274,7 @@ def make_nsi( 'PRE_UNINSTALL': '@pre_uninstall.bat', 'INDEX_CACHE': '@cache', 'REPODATA_RECORD': '@repodata_record.json', + 'VIRTUAL_SPECS': " ".join([f'"{spec}"' for spec in info.get("virtual_specs", ())]) } # These are NSIS predefines and must not be replaced diff --git a/examples/virtual_specs/construct.yaml b/examples/virtual_specs/construct.yaml new file mode 100644 index 000000000..e5c62da32 --- /dev/null +++ b/examples/virtual_specs/construct.yaml @@ -0,0 +1,20 @@ +name: virtual_specs + +version: 0.0.1 + +keep_pkgs: True + +channels: + - conda-forge + +specs: + - ca-certificates + +virtual_specs: + - __osx>=20 # [osx] + - __glibc>=20 # [linux] + - __win<0 # [win] + +initialize_by_default: false +register_python: false +installer_type: all diff --git a/tests/test_examples.py b/tests/test_examples.py index 66edb7895..97768d4f5 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -662,3 +662,11 @@ def test_cross_osx_building(tmp_path): extra_constructor_args=["--platform", "osx-arm64"], config_filename="constructor_input.yaml", ) + + +def test_virtual_specs(tmp_path, request): + input_path = _example_path("virtual_specs") + for installer, install_dir in create_installer(input_path, tmp_path): + with pytest.raises(subprocess.CalledProcessError): + # This example is configured to fail due to unsatisfiable virtual specs + _run_installer(input_path, installer, install_dir, request=request) From 65ddaba24ac1f372b41cd3a2f9aee704e336c171 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 11:13:50 +0200 Subject: [PATCH 02/34] ignore shellcheck here --- constructor/header.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/constructor/header.sh b/constructor/header.sh index da6132c43..2249870a7 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -424,6 +424,7 @@ export TMP="$PREFIX/install_tmp" mkdir -p "$TMP" # Check whether the virtual specs can be satisfied +# shellcheck disable=SC2050 if [ "__VIRTUAL_SPECS__" != "" ]; then CONDA_QUIET="$BATCH" \ CONDA_SOLVER="classic" \ From 39b61cd25bdf7af48e1a4bf9d7d26ddc3a66f930 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 11:18:50 +0200 Subject: [PATCH 03/34] run inside PKG too (for cases other than __osx) --- constructor/osx/prepare_installation.sh | 8 ++++++++ constructor/osxpkg.py | 2 ++ 2 files changed, 10 insertions(+) diff --git a/constructor/osx/prepare_installation.sh b/constructor/osx/prepare_installation.sh index 93e34fcea..512a77fe3 100644 --- a/constructor/osx/prepare_installation.sh +++ b/constructor/osx/prepare_installation.sh @@ -31,6 +31,14 @@ chmod +x "$CONDA_EXEC" mkdir -p "$PREFIX/conda-meta" touch "$PREFIX/conda-meta/history" +# Check whether the virtual specs can be satisfied +# shellcheck disable=SC2050 +if [ "__VIRTUAL_SPECS__" != "" ]; then + CONDA_QUIET="$BATCH" \ + CONDA_SOLVER="classic" \ + "$CONDA_EXEC" create --dry-run --prefix "$PREFIX" --offline __VIRTUAL_SPECS__ +fi + # Create $PREFIX/.nonadmin if the installation didn't require superuser permissions if [ "$(id -u)" -ne 0 ]; then touch "$PREFIX/.nonadmin" diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index d0951d5eb..46c7cd2eb 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -1,5 +1,6 @@ import logging import os +import shlex import shutil import sys import xml.etree.ElementTree as ET @@ -346,6 +347,7 @@ def move_script(src, dst, info, ensure_shebang=False, user_script_type=None): 'SHORTCUTS': shortcuts_flags(info), 'ENABLE_SHORTCUTS': str(info['_enable_shortcuts']).lower(), 'REGISTER_ENVS': str(info.get("register_envs", True)).lower(), + 'VIRTUAL_SPECS': shlex.join(info.get("virtual_specs", ())), } data = preprocess(data, ppd) custom_variables = info.get('script_env_variables', {}) From 75dfbfaa9f1b8c451985cb9b5d853872ffe8da0c Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 11:19:30 +0200 Subject: [PATCH 04/34] pre-commit --- constructor/osxpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index 46c7cd2eb..b60b64ff3 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -10,7 +10,7 @@ from tempfile import NamedTemporaryFile from . import preconda -from .conda_interface import conda_context, MatchSpec +from .conda_interface import MatchSpec, conda_context from .construct import ns_platform, parse from .imaging import write_images from .utils import ( From b6c42339503102cf228269d4fefd4009ae4dcd47 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 11:20:52 +0200 Subject: [PATCH 05/34] add news --- news/809-virtual-specs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 news/809-virtual-specs diff --git a/news/809-virtual-specs b/news/809-virtual-specs new file mode 100644 index 000000000..9bfe6b04e --- /dev/null +++ b/news/809-virtual-specs @@ -0,0 +1,19 @@ +### Enhancements + +* A new setting `virtual_specs` allows the installer to run some solver checks before the installation proceeds. Useful for checking whether certain virtual package versions can be satisfied. (#809) + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* From d9f7133a6507fb0a6d980829fa8d187ff79921da Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 11:34:07 +0200 Subject: [PATCH 06/34] update docs --- CONSTRUCT.md | 8 ++++++++ docs/source/construct-yaml.md | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/CONSTRUCT.md b/CONSTRUCT.md index 9e02ff8e2..82e7997e3 100644 --- a/CONSTRUCT.md +++ b/CONSTRUCT.md @@ -113,6 +113,14 @@ for example, if `python=3.6` is included, then conda will always seek versions of packages compatible with Python 3.6. If this is option is not provided, it will be set equal to the value of `specs`. +### `virtual_specs` + +_required:_ no
+_type:_ list
+ +A list of virtual packages that must be satisfied at install time.Virtual +packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. + ### `exclude` _required:_ no
diff --git a/docs/source/construct-yaml.md b/docs/source/construct-yaml.md index 9e02ff8e2..82e7997e3 100644 --- a/docs/source/construct-yaml.md +++ b/docs/source/construct-yaml.md @@ -113,6 +113,14 @@ for example, if `python=3.6` is included, then conda will always seek versions of packages compatible with Python 3.6. If this is option is not provided, it will be set equal to the value of `specs`. +### `virtual_specs` + +_required:_ no
+_type:_ list
+ +A list of virtual packages that must be satisfied at install time.Virtual +packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. + ### `exclude` _required:_ no
From a017d7d926c57f67c465d244e454e2ef6bcd832c Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 11:34:33 +0200 Subject: [PATCH 07/34] reveal exceptions just to make sure --- tests/test_examples.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 97768d4f5..53c5a4726 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -667,6 +667,6 @@ def test_cross_osx_building(tmp_path): def test_virtual_specs(tmp_path, request): input_path = _example_path("virtual_specs") for installer, install_dir in create_installer(input_path, tmp_path): - with pytest.raises(subprocess.CalledProcessError): + # with pytest.raises(subprocess.CalledProcessError): # This example is configured to fail due to unsatisfiable virtual specs - _run_installer(input_path, installer, install_dir, request=request) + _run_installer(input_path, installer, install_dir, request=request) From 220f7620919bf2e05d6ee8b8a45300a9bb088a56 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 12:13:57 +0200 Subject: [PATCH 08/34] Make sure we fail for the right reason --- constructor/nsis/main.nsi.tmpl | 2 +- tests/test_examples.py | 60 ++++++++++++++++++++++++---------- 2 files changed, 44 insertions(+), 18 deletions(-) diff --git a/constructor/nsis/main.nsi.tmpl b/constructor/nsis/main.nsi.tmpl index 28332462a..2c7fb6951 100644 --- a/constructor/nsis/main.nsi.tmpl +++ b/constructor/nsis/main.nsi.tmpl @@ -1137,7 +1137,7 @@ Section "Install" SetDetailsPrint TextOnly DetailPrint "Checking virtual specs..." push '"$INSTDIR\_conda.exe" create --dry-run --prefix "$INSTDIR" --offline __VIRTUAL_SPECS__' - push 'Virtual specs checks failed' + push 'Failed to check virtual specs: __VIRTUAL_SPECS__' push 'WithLog' call AbortRetryNSExecWait SetDetailsPrint both diff --git a/tests/test_examples.py b/tests/test_examples.py index 53c5a4726..e722d0815 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -102,7 +102,7 @@ def _check_installer_log(install_dir): raise AssertionError("\n".join(error_lines)) -def _run_installer_exe(installer, install_dir, installer_input=None, timeout=420): +def _run_installer_exe(installer, install_dir, installer_input=None, timeout=420, check=True): """ NSIS manual: > /D sets the default installation directory ($INSTDIR), overriding InstallDir @@ -125,11 +125,12 @@ def _run_installer_exe(installer, install_dir, installer_input=None, timeout=420 "after completion." ) cmd = ["cmd.exe", "/c", "start", "/wait", installer, "/S", *f"/D={install_dir}".split()] - _execute(cmd, installer_input=installer_input, timeout=timeout) - _check_installer_log(install_dir) + _execute(cmd, installer_input=installer_input, timeout=timeout, check=check) + if check: + _check_installer_log(install_dir) -def _run_uninstaller_exe(install_dir, timeout=420): +def _run_uninstaller_exe(install_dir, timeout=420, check=True): # Now test the uninstallers if " " in str(install_dir): # TODO: We can't seem to run the uninstaller when there are spaces in the PATH @@ -158,7 +159,7 @@ def _run_uninstaller_exe(install_dir, timeout=420): # us problems with the tempdir cleanup later f"/S _?={install_dir}", ] - _execute(cmd, timeout=timeout) + _execute(cmd, timeout=timeout, check=check) _check_installer_log(install_dir) remaining_files = list(install_dir.iterdir()) if len(remaining_files) > 3: @@ -171,12 +172,12 @@ def _run_uninstaller_exe(install_dir, timeout=420): raise AssertionError(f"Uninstaller left too many files: {remaining_files}") -def _run_installer_sh(installer, install_dir, installer_input=None, timeout=420): +def _run_installer_sh(installer, install_dir, installer_input=None, timeout=420, check=True): if installer_input: cmd = ["/bin/sh", installer] else: cmd = ["/bin/sh", installer, "-b", "-p", install_dir] - return _execute(cmd, installer_input=installer_input, timeout=timeout) + return _execute(cmd, installer_input=installer_input, timeout=timeout, check=check) def _run_installer_pkg( @@ -185,6 +186,7 @@ def _run_installer_pkg( example_path=None, config_filename="construct.yaml", timeout=420, + check=True, ): if os.environ.get("CI"): # We want to run it in an arbitrary directory, but the options @@ -208,7 +210,7 @@ def _run_installer_pkg( "Export CI=1 to run it, but it will pollute your $HOME." ) cmd = ["pkgutil", "--expand", installer, install_dir] - return _execute(cmd, timeout=timeout), install_dir + return _execute(cmd, timeout=timeout, check=check), install_dir def _sentinel_file_checks(example_path, install_dir): @@ -230,14 +232,27 @@ def _run_installer( installer_input: Optional[str] = None, config_filename="construct.yaml", check_sentinels=True, + check_subprocess=True, request=None, uninstall=True, timeout=420, ): if installer.suffix == ".exe": - _run_installer_exe(installer, install_dir, installer_input=installer_input, timeout=timeout) + _run_installer_exe( + installer, + install_dir, + installer_input=installer_input, + timeout=timeout, + check=check_subprocess, + ) elif installer.suffix == ".sh": - _run_installer_sh(installer, install_dir, installer_input=installer_input, timeout=timeout) + _run_installer_sh( + installer, + install_dir, + installer_input=installer_input, + timeout=timeout, + check=check_subprocess, + ) elif installer.suffix == ".pkg": if request and ON_CI: request.addfinalizer(lambda: shutil.rmtree(str(install_dir))) @@ -247,6 +262,7 @@ def _run_installer( example_path=example_path, config_filename=config_filename, timeout=timeout, + check=check_subprocess, ) else: raise ValueError(f"Unknown installer type: {installer.suffix}") @@ -511,12 +527,12 @@ def test_example_signing(tmp_path, request): @pytest.mark.skipif(sys.platform != "win32", reason="Windows only") @pytest.mark.skipif( - not shutil.which("azuresigntool") and not os.environ.get("AZURE_SIGNTOOL_PATH"), - reason="AzureSignTool not available" + not shutil.which("azuresigntool") and not os.environ.get("AZURE_SIGNTOOL_PATH"), + reason="AzureSignTool not available", ) @pytest.mark.parametrize( - "auth_method", - os.environ.get("AZURE_SIGNTOOL_TEST_AUTH_METHODS", "token,secret").split(","), + "auth_method", + os.environ.get("AZURE_SIGNTOOL_TEST_AUTH_METHODS", "token,secret").split(","), ) def test_azure_signtool(tmp_path, request, monkeypatch, auth_method): """Test signing installers with AzureSignTool. @@ -667,6 +683,16 @@ def test_cross_osx_building(tmp_path): def test_virtual_specs(tmp_path, request): input_path = _example_path("virtual_specs") for installer, install_dir in create_installer(input_path, tmp_path): - # with pytest.raises(subprocess.CalledProcessError): - # This example is configured to fail due to unsatisfiable virtual specs - _run_installer(input_path, installer, install_dir, request=request) + process = _run_installer( + input_path, installer, install_dir, request=request, check_subprocess=False + ) + # This example is configured to fail due to unsatisfiable virtual specs + assert process.returncode != 0 + if installer.suffix == ".exe": + with pytest.raises(AssertionError, match="Failed to check virtual specs"): + _check_installer_log(install_dir) + elif CONDA_EXE == "micromamba": + msg = "is missing on the system" + else: + msg = "PackagesNotFoundError" + assert msg in process.stdout + process.stderr From 2960219e4d811b82e54e20b111bd4485e928b720 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 12:33:29 +0200 Subject: [PATCH 09/34] return the process --- tests/test_examples.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index e722d0815..79a242b51 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -159,17 +159,19 @@ def _run_uninstaller_exe(install_dir, timeout=420, check=True): # us problems with the tempdir cleanup later f"/S _?={install_dir}", ] - _execute(cmd, timeout=timeout, check=check) - _check_installer_log(install_dir) - remaining_files = list(install_dir.iterdir()) - if len(remaining_files) > 3: - # The debug installer writes to install.log too, which will only - # be deleted _after_ a reboot. Finding some files is ok, but more - # than two usually means a problem with the uninstaller. - # Note this is is not exhaustive, because we are not checking - # whether the registry was restored, menu items were deleted, etc. - # TODO :) - raise AssertionError(f"Uninstaller left too many files: {remaining_files}") + process = _execute(cmd, timeout=timeout, check=check) + if check: + _check_installer_log(install_dir) + remaining_files = list(install_dir.iterdir()) + if len(remaining_files) > 3: + # The debug installer writes to install.log too, which will only + # be deleted _after_ a reboot. Finding some files is ok, but more + # than two usually means a problem with the uninstaller. + # Note this is is not exhaustive, because we are not checking + # whether the registry was restored, menu items were deleted, etc. + # TODO :) + raise AssertionError(f"Uninstaller left too many files: {remaining_files}") + return process def _run_installer_sh(installer, install_dir, installer_input=None, timeout=420, check=True): @@ -236,9 +238,9 @@ def _run_installer( request=None, uninstall=True, timeout=420, -): +) -> subprocess.CompletedProcess: if installer.suffix == ".exe": - _run_installer_exe( + process = _run_installer_exe( installer, install_dir, installer_input=installer_input, @@ -246,7 +248,7 @@ def _run_installer( check=check_subprocess, ) elif installer.suffix == ".sh": - _run_installer_sh( + process = _run_installer_sh( installer, install_dir, installer_input=installer_input, @@ -256,7 +258,7 @@ def _run_installer( elif installer.suffix == ".pkg": if request and ON_CI: request.addfinalizer(lambda: shutil.rmtree(str(install_dir))) - _run_installer_pkg( + process = _run_installer_pkg( installer, install_dir, example_path=example_path, @@ -269,7 +271,8 @@ def _run_installer( if check_sentinels: _sentinel_file_checks(example_path, install_dir) if uninstall and installer.suffix == ".exe": - _run_uninstaller_exe(install_dir, timeout=timeout) + _run_uninstaller_exe(install_dir, timeout=timeout, check=check_subprocess) + return process def create_installer( From b36495f5bf2214481bc0c77074600b443fa322b4 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 13:03:33 +0200 Subject: [PATCH 10/34] ignore errors, process tuple --- tests/test_examples.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 79a242b51..b4fe97889 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -257,8 +257,8 @@ def _run_installer( ) elif installer.suffix == ".pkg": if request and ON_CI: - request.addfinalizer(lambda: shutil.rmtree(str(install_dir))) - process = _run_installer_pkg( + request.addfinalizer(lambda: shutil.rmtree(str(install_dir), ignore_errors=True)) + process, _ = _run_installer_pkg( installer, install_dir, example_path=example_path, From 26ccbe40ee5ae550787bbfed0be246af09551740 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 13:24:33 +0200 Subject: [PATCH 11/34] fix test on windows --- examples/virtual_specs/construct.yaml | 2 ++ tests/test_examples.py | 15 +++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/examples/virtual_specs/construct.yaml b/examples/virtual_specs/construct.yaml index e5c62da32..9646044e3 100644 --- a/examples/virtual_specs/construct.yaml +++ b/examples/virtual_specs/construct.yaml @@ -17,4 +17,6 @@ virtual_specs: initialize_by_default: false register_python: false +check_path_spaces: false +check_path_length: false installer_type: all diff --git a/tests/test_examples.py b/tests/test_examples.py index b4fe97889..efe14fcf4 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -125,9 +125,10 @@ def _run_installer_exe(installer, install_dir, installer_input=None, timeout=420 "after completion." ) cmd = ["cmd.exe", "/c", "start", "/wait", installer, "/S", *f"/D={install_dir}".split()] - _execute(cmd, installer_input=installer_input, timeout=timeout, check=check) + process = _execute(cmd, installer_input=installer_input, timeout=timeout, check=check) if check: _check_installer_log(install_dir) + return process def _run_uninstaller_exe(install_dir, timeout=420, check=True): @@ -687,15 +688,21 @@ def test_virtual_specs(tmp_path, request): input_path = _example_path("virtual_specs") for installer, install_dir in create_installer(input_path, tmp_path): process = _run_installer( - input_path, installer, install_dir, request=request, check_subprocess=False + input_path, + installer, + install_dir, + request=request, + check_subprocess=False, + uninstall=False, ) # This example is configured to fail due to unsatisfiable virtual specs - assert process.returncode != 0 if installer.suffix == ".exe": with pytest.raises(AssertionError, match="Failed to check virtual specs"): _check_installer_log(install_dir) - elif CONDA_EXE == "micromamba": + continue + if CONDA_EXE == "micromamba": msg = "is missing on the system" else: msg = "PackagesNotFoundError" + assert process.returncode != 0 assert msg in process.stdout + process.stderr From 39f689df7539a2584e98de2ff6de4e3fa0c6609e Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 13:51:07 +0200 Subject: [PATCH 12/34] Pass as unescaped variable --- constructor/nsis/main.nsi.tmpl | 6 +++--- constructor/winexe.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/constructor/nsis/main.nsi.tmpl b/constructor/nsis/main.nsi.tmpl index 2c7fb6951..a32e6e506 100644 --- a/constructor/nsis/main.nsi.tmpl +++ b/constructor/nsis/main.nsi.tmpl @@ -1132,12 +1132,12 @@ Section "Install" System::Call 'kernel32::SetEnvironmentVariable(t,t)i("INSTALLER_PLAT", "${PLATFORM}").r0' System::Call 'kernel32::SetEnvironmentVariable(t,t)i("INSTALLER_TYPE", "EXE").r0' - ${If} '__VIRTUAL_SPECS__' != '' + ${If} '@VIRTUAL_SPECS@' != '' System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_SOLVER", "classic").r0' SetDetailsPrint TextOnly DetailPrint "Checking virtual specs..." - push '"$INSTDIR\_conda.exe" create --dry-run --prefix "$INSTDIR" --offline __VIRTUAL_SPECS__' - push 'Failed to check virtual specs: __VIRTUAL_SPECS__' + push '"$INSTDIR\_conda.exe" create --dry-run --prefix "$INSTDIR" --offline @VIRTUAL_SPECS@' + push 'Failed to check virtual specs: @VIRTUAL_SPECS@' push 'WithLog' call AbortRetryNSExecWait SetDetailsPrint both diff --git a/constructor/winexe.py b/constructor/winexe.py index 9412c4223..cacdca569 100644 --- a/constructor/winexe.py +++ b/constructor/winexe.py @@ -274,7 +274,6 @@ def make_nsi( 'PRE_UNINSTALL': '@pre_uninstall.bat', 'INDEX_CACHE': '@cache', 'REPODATA_RECORD': '@repodata_record.json', - 'VIRTUAL_SPECS': " ".join([f'"{spec}"' for spec in info.get("virtual_specs", ())]) } # These are NSIS predefines and must not be replaced @@ -375,6 +374,8 @@ def make_nsi( else '' ), ('@TEMP_EXTRA_FILES@', '\n '.join(insert_tempfiles_commands(temp_extra_files))), + ('@VIRTUAL_SPECS@', " ".join([f'"{spec}"' for spec in info.get("virtual_specs", ())])), + ]: data = data.replace(key, value) From 3154b02dc74466cea3fe6e9af74188a82dfa9f5d Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 21 Jun 2024 16:52:41 +0200 Subject: [PATCH 13/34] adjust error message for .pkg --- constructor/osxpkg.py | 4 +++- tests/test_examples.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index b60b64ff3..255d37725 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -206,7 +206,9 @@ def modify_xml(xml_path, info): if osx_versions: allowed_os_versions = ET.Element("allowed-os-versions") allowed_os_versions.append(ET.Element("os-version", osx_versions)) - root.append(allowed_os_versions) + volume_check = ET.Element("volume-check") + volume_check.append(allowed_os_versions) + root.append(volume_check) # See below for an explanation of the consequences of this # customLocation value. diff --git a/tests/test_examples.py b/tests/test_examples.py index efe14fcf4..e3cf065b3 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -700,7 +700,11 @@ def test_virtual_specs(tmp_path, request): with pytest.raises(AssertionError, match="Failed to check virtual specs"): _check_installer_log(install_dir) continue - if CONDA_EXE == "micromamba": + elif installer.suffix == ".pkg": + # The GUI does provide a better message with the min version and so on + # but on the CLI we fail with this one instead + msg = "Cannot install on volume" + elif CONDA_EXE == "micromamba": msg = "is missing on the system" else: msg = "PackagesNotFoundError" From 247bd45dd88ecc81212d7fa9c7977afbc375cf77 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 25 Jun 2024 15:10:23 +0200 Subject: [PATCH 14/34] Adjust docs --- CONSTRUCT.md | 6 +++++- constructor/construct.py | 6 +++++- docs/source/construct-yaml.md | 6 +++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CONSTRUCT.md b/CONSTRUCT.md index 82e7997e3..e67b3f640 100644 --- a/CONSTRUCT.md +++ b/CONSTRUCT.md @@ -118,8 +118,12 @@ will be set equal to the value of `specs`. _required:_ no
_type:_ list
-A list of virtual packages that must be satisfied at install time.Virtual +A list of virtual packages that must be satisfied at install time. Virtual packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. +These specs are dry-run solved offline by the bundled `--conda-exe` binary. +In PKG installers, `__osx` specs can be checked natively without the solver +being involved as long as: the version constraint is single-component and only +uses the operator `>=` or `<=`. ### `exclude` diff --git a/constructor/construct.py b/constructor/construct.py index e5c33b4fd..9b49e7259 100644 --- a/constructor/construct.py +++ b/constructor/construct.py @@ -84,8 +84,12 @@ '''), ('virtual_specs', False, list, ''' -A list of virtual packages that must be satisfied at install time.Virtual +A list of virtual packages that must be satisfied at install time. Virtual packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. +These specs are dry-run solved offline by the bundled `--conda-exe` binary. +In PKG installers, `__osx` specs can be checked natively without the solver +being involved as long as: the version constraint is single-component and only +uses the operator `>=` or `<=`. '''), ('exclude', False, list, ''' diff --git a/docs/source/construct-yaml.md b/docs/source/construct-yaml.md index 82e7997e3..e67b3f640 100644 --- a/docs/source/construct-yaml.md +++ b/docs/source/construct-yaml.md @@ -118,8 +118,12 @@ will be set equal to the value of `specs`. _required:_ no
_type:_ list
-A list of virtual packages that must be satisfied at install time.Virtual +A list of virtual packages that must be satisfied at install time. Virtual packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. +These specs are dry-run solved offline by the bundled `--conda-exe` binary. +In PKG installers, `__osx` specs can be checked natively without the solver +being involved as long as: the version constraint is single-component and only +uses the operator `>=` or `<=`. ### `exclude` From fd55c515e8f46dea3d2003c8ff104a8f6004c8df Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 25 Jun 2024 15:12:59 +0200 Subject: [PATCH 15/34] validate virtual_specs --- constructor/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/constructor/main.py b/constructor/main.py index 346882a95..9f538cfb8 100644 --- a/constructor/main.py +++ b/constructor/main.py @@ -120,6 +120,13 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, continue if isinstance(info[key], str): info[key] = list(yield_lines(join(dir_path, info[key]))) + if key == "virtual_specs": + for value in info[key]: + if not value.starswith("__"): + raise ValueError( + "'virtual_specs' can only include virtual package names like '__name', " + f"but you supplied: {value}." + ) # normalize paths to be copied; if they are relative, they must be to # construct.yaml's parent (dir_path) From 5399871e6799e6eb7aedad37f9278f0d843f33d9 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 25 Jun 2024 15:13:59 +0200 Subject: [PATCH 16/34] use <= for osx --- examples/virtual_specs/construct.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/virtual_specs/construct.yaml b/examples/virtual_specs/construct.yaml index 9646044e3..9adf4c2eb 100644 --- a/examples/virtual_specs/construct.yaml +++ b/examples/virtual_specs/construct.yaml @@ -11,7 +11,7 @@ specs: - ca-certificates virtual_specs: - - __osx>=20 # [osx] + - __osx<=9 # [osx] - __glibc>=20 # [linux] - __win<0 # [win] From ec575b682f196332de27fba99ec7c122c832ca8a Mon Sep 17 00:00:00 2001 From: jaimergp Date: Tue, 25 Jun 2024 15:16:24 +0200 Subject: [PATCH 17/34] add comment about solver=classic --- constructor/header.sh | 3 +++ constructor/nsis/main.nsi.tmpl | 3 +++ constructor/osx/prepare_installation.sh | 3 +++ 3 files changed, 9 insertions(+) diff --git a/constructor/header.sh b/constructor/header.sh index 2249870a7..ed88fae5a 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -424,6 +424,9 @@ export TMP="$PREFIX/install_tmp" mkdir -p "$TMP" # Check whether the virtual specs can be satisfied +# We need to specify CONDA_SOLVER=classic for conda-standalone +# to work around this bug in conda-libmamba-solver: +# https://github.com/conda/conda-libmamba-solver/issues/480 # shellcheck disable=SC2050 if [ "__VIRTUAL_SPECS__" != "" ]; then CONDA_QUIET="$BATCH" \ diff --git a/constructor/nsis/main.nsi.tmpl b/constructor/nsis/main.nsi.tmpl index a32e6e506..96190cf6e 100644 --- a/constructor/nsis/main.nsi.tmpl +++ b/constructor/nsis/main.nsi.tmpl @@ -1133,6 +1133,9 @@ Section "Install" System::Call 'kernel32::SetEnvironmentVariable(t,t)i("INSTALLER_TYPE", "EXE").r0' ${If} '@VIRTUAL_SPECS@' != '' + # We need to specify CONDA_SOLVER=classic for conda-standalone + # to work around this bug in conda-libmamba-solver: + # https://github.com/conda/conda-libmamba-solver/issues/480 System::Call 'kernel32::SetEnvironmentVariable(t,t)i("CONDA_SOLVER", "classic").r0' SetDetailsPrint TextOnly DetailPrint "Checking virtual specs..." diff --git a/constructor/osx/prepare_installation.sh b/constructor/osx/prepare_installation.sh index 512a77fe3..6d59d60a8 100644 --- a/constructor/osx/prepare_installation.sh +++ b/constructor/osx/prepare_installation.sh @@ -32,6 +32,9 @@ mkdir -p "$PREFIX/conda-meta" touch "$PREFIX/conda-meta/history" # Check whether the virtual specs can be satisfied +# We need to specify CONDA_SOLVER=classic for conda-standalone +# to work around this bug in conda-libmamba-solver: +# https://github.com/conda/conda-libmamba-solver/issues/480 # shellcheck disable=SC2050 if [ "__VIRTUAL_SPECS__" != "" ]; then CONDA_QUIET="$BATCH" \ From df02fbd9646a9963422850c1b1721493cacf34c6 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Wed, 26 Jun 2024 11:31:47 +0200 Subject: [PATCH 18/34] typo --- constructor/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constructor/main.py b/constructor/main.py index 9f538cfb8..4625a9a69 100644 --- a/constructor/main.py +++ b/constructor/main.py @@ -122,7 +122,7 @@ def main_build(dir_path, output_dir='.', platform=cc_platform, info[key] = list(yield_lines(join(dir_path, info[key]))) if key == "virtual_specs": for value in info[key]: - if not value.starswith("__"): + if not value.startswith("__"): raise ValueError( "'virtual_specs' can only include virtual package names like '__name', " f"but you supplied: {value}." From 860066697b2bb005fce05da941a426c1994f9126 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 12 Jul 2024 15:11:07 +0200 Subject: [PATCH 19/34] Parse , (AND) in __osx Co-authored-by: Marco Esters --- CONSTRUCT.md | 3 +-- constructor/construct.py | 3 +-- constructor/osxpkg.py | 17 ++++++++++++----- docs/source/construct-yaml.md | 3 +-- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CONSTRUCT.md b/CONSTRUCT.md index e67b3f640..ab0bca116 100644 --- a/CONSTRUCT.md +++ b/CONSTRUCT.md @@ -122,8 +122,7 @@ A list of virtual packages that must be satisfied at install time. Virtual packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. These specs are dry-run solved offline by the bundled `--conda-exe` binary. In PKG installers, `__osx` specs can be checked natively without the solver -being involved as long as: the version constraint is single-component and only -uses the operator `>=` or `<=`. +being involved as long as only `>=`, `<=` or `,` are used. ### `exclude` diff --git a/constructor/construct.py b/constructor/construct.py index 9b49e7259..fb6ead2f4 100644 --- a/constructor/construct.py +++ b/constructor/construct.py @@ -88,8 +88,7 @@ packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. These specs are dry-run solved offline by the bundled `--conda-exe` binary. In PKG installers, `__osx` specs can be checked natively without the solver -being involved as long as: the version constraint is single-component and only -uses the operator `>=` or `<=`. +being involved as long as only `>=`, `<=` or `,` are used. '''), ('exclude', False, list, ''' diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index 255d37725..22509ab1d 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -197,11 +197,18 @@ def modify_xml(xml_path, info): continue if not spec.version: continue - operator = spec.version.operator_func.__name__ - if operator == "ge": - osx_versions["min"] = str(spec.version.matcher_vo) - elif operator == "le": - osx_versions["max"] = str(spec.version.matcher_vo) + if "|" in spec.version.spec_str: + raise ValueError("Can't process `|`-joined versions. Only `,` is allowed.") + versions = spec.version.tup if "," in spec.version.spec_str else (spec.version,) + for version in versions: + operator = version.operator_func.__name__ + if operator == "ge": + osx_versions["min"] = str(version.matcher_vo) + elif operator == "le": + osx_versions["max"] = str(version.matcher_vo) + else: + raise ValueError( + f"Invalid version operator for {spec}. Only `<=` or `>=` are supported.") if osx_versions: allowed_os_versions = ET.Element("allowed-os-versions") diff --git a/docs/source/construct-yaml.md b/docs/source/construct-yaml.md index e67b3f640..ab0bca116 100644 --- a/docs/source/construct-yaml.md +++ b/docs/source/construct-yaml.md @@ -122,8 +122,7 @@ A list of virtual packages that must be satisfied at install time. Virtual packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. These specs are dry-run solved offline by the bundled `--conda-exe` binary. In PKG installers, `__osx` specs can be checked natively without the solver -being involved as long as: the version constraint is single-component and only -uses the operator `>=` or `<=`. +being involved as long as only `>=`, `<=` or `,` are used. ### `exclude` From adcbab9a31c9d7cf2d6ac599d2b73d3bb6afb4f7 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 12 Jul 2024 15:53:19 +0200 Subject: [PATCH 20/34] Fix PKG os-version control --- CONSTRUCT.md | 2 +- constructor/construct.py | 2 +- constructor/osxpkg.py | 10 +++++++--- docs/source/construct-yaml.md | 2 +- examples/virtual_specs/construct.yaml | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CONSTRUCT.md b/CONSTRUCT.md index ab0bca116..d15f5484b 100644 --- a/CONSTRUCT.md +++ b/CONSTRUCT.md @@ -122,7 +122,7 @@ A list of virtual packages that must be satisfied at install time. Virtual packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. These specs are dry-run solved offline by the bundled `--conda-exe` binary. In PKG installers, `__osx` specs can be checked natively without the solver -being involved as long as only `>=`, `<=` or `,` are used. +being involved as long as only `>=`, `<` or `,` are used. ### `exclude` diff --git a/constructor/construct.py b/constructor/construct.py index fb6ead2f4..515392a4a 100644 --- a/constructor/construct.py +++ b/constructor/construct.py @@ -88,7 +88,7 @@ packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. These specs are dry-run solved offline by the bundled `--conda-exe` binary. In PKG installers, `__osx` specs can be checked natively without the solver -being involved as long as only `>=`, `<=` or `,` are used. +being involved as long as only `>=`, `<` or `,` are used. '''), ('exclude', False, list, ''' diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index 22509ab1d..742f2b4c9 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -190,6 +190,7 @@ def modify_xml(xml_path, info): root.append(readme) # -- __osx virtual package checks -- # + # Reference: https://developer.apple.com/library/archive/documentation/DeveloperTools/Reference/DistributionDefinitionRef/Chapters/Distribution_XML_Ref.html osx_versions = {} for spec in info.get("virtual_specs", ()): spec = MatchSpec(spec) @@ -204,13 +205,16 @@ def modify_xml(xml_path, info): operator = version.operator_func.__name__ if operator == "ge": osx_versions["min"] = str(version.matcher_vo) - elif operator == "le": - osx_versions["max"] = str(version.matcher_vo) + elif operator == "lt": + osx_versions["before"] = str(version.matcher_vo) else: raise ValueError( - f"Invalid version operator for {spec}. Only `<=` or `>=` are supported.") + f"Invalid version operator for {spec}. Only `<` or `>=` are supported." + ) if osx_versions: + if "min" not in osx_versions: + raise ValueError("Specifying __osx requires a lower bound with `>=`") allowed_os_versions = ET.Element("allowed-os-versions") allowed_os_versions.append(ET.Element("os-version", osx_versions)) volume_check = ET.Element("volume-check") diff --git a/docs/source/construct-yaml.md b/docs/source/construct-yaml.md index ab0bca116..d15f5484b 100644 --- a/docs/source/construct-yaml.md +++ b/docs/source/construct-yaml.md @@ -122,7 +122,7 @@ A list of virtual packages that must be satisfied at install time. Virtual packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. These specs are dry-run solved offline by the bundled `--conda-exe` binary. In PKG installers, `__osx` specs can be checked natively without the solver -being involved as long as only `>=`, `<=` or `,` are used. +being involved as long as only `>=`, `<` or `,` are used. ### `exclude` diff --git a/examples/virtual_specs/construct.yaml b/examples/virtual_specs/construct.yaml index 9adf4c2eb..f387ee2fc 100644 --- a/examples/virtual_specs/construct.yaml +++ b/examples/virtual_specs/construct.yaml @@ -11,7 +11,7 @@ specs: - ca-certificates virtual_specs: - - __osx<=9 # [osx] + - __osx>=5.1,<9 # [osx] - __glibc>=20 # [linux] - __win<0 # [win] From 263bd0e10b56db6f54e9c67de579cbcb8af853d2 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Fri, 12 Jul 2024 15:54:05 +0200 Subject: [PATCH 21/34] pre-commit --- constructor/osxpkg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index 742f2b4c9..a514e351c 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -190,7 +190,7 @@ def modify_xml(xml_path, info): root.append(readme) # -- __osx virtual package checks -- # - # Reference: https://developer.apple.com/library/archive/documentation/DeveloperTools/Reference/DistributionDefinitionRef/Chapters/Distribution_XML_Ref.html + # Reference: https://developer.apple.com/library/archive/documentation/DeveloperTools/Reference/DistributionDefinitionRef/Chapters/Distribution_XML_Ref.html # noqa osx_versions = {} for spec in info.get("virtual_specs", ()): spec = MatchSpec(spec) From b17085e3058481c9cb18c1168a4c3afee5c6b166 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 25 Jul 2024 12:01:47 +0200 Subject: [PATCH 22/34] Add Bash-only checks for SH installers --- CONSTRUCT.md | 7 ++++--- constructor/construct.py | 7 ++++--- constructor/header.sh | 18 ++++++++++++++++++ constructor/osxpkg.py | 25 +++---------------------- constructor/shar.py | 7 +++++++ constructor/utils.py | 27 +++++++++++++++++++++++++++ docs/source/construct-yaml.md | 7 ++++--- examples/virtual_specs/construct.yaml | 6 +++--- 8 files changed, 70 insertions(+), 34 deletions(-) diff --git a/CONSTRUCT.md b/CONSTRUCT.md index d15f5484b..67114e00d 100644 --- a/CONSTRUCT.md +++ b/CONSTRUCT.md @@ -121,8 +121,9 @@ _type:_ list
A list of virtual packages that must be satisfied at install time. Virtual packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. These specs are dry-run solved offline by the bundled `--conda-exe` binary. -In PKG installers, `__osx` specs can be checked natively without the solver -being involved as long as only `>=`, `<` or `,` are used. +In SH installers, `__glibc>=x.y` and `__osx>=x.y` specs can be checked with +Bash only. In PKG installers, `__osx` specs can be checked natively without +the solver being involved as long as only `>=`, `<` or `,` are used. ### `exclude` @@ -821,7 +822,7 @@ _required:_ no
_type:_ list
Temporary files that could be referenced in the installation process (i.e. customized - `welcome_file` and `conclusion_file` (see above)) . Should be a list of +`welcome_file` and `conclusion_file` (see above)) . Should be a list of file paths, relative to the directory where `construct.yaml` is. In Windows, these files will be copied into a temporary folder, the NSIS `$PLUGINSDIR`, during install process (Windows only). diff --git a/constructor/construct.py b/constructor/construct.py index 515392a4a..c6b8de183 100644 --- a/constructor/construct.py +++ b/constructor/construct.py @@ -87,8 +87,9 @@ A list of virtual packages that must be satisfied at install time. Virtual packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. These specs are dry-run solved offline by the bundled `--conda-exe` binary. -In PKG installers, `__osx` specs can be checked natively without the solver -being involved as long as only `>=`, `<` or `,` are used. +In SH installers, `__glibc>=x.y` and `__osx>=x.y` specs can be checked with +Bash only. In PKG installers, `__osx` specs can be checked natively without +the solver being involved as long as only `>=`, `<` or `,` are used. '''), ('exclude', False, list, ''' @@ -602,7 +603,7 @@ ('temp_extra_files', False, list, ''' Temporary files that could be referenced in the installation process (i.e. customized - `welcome_file` and `conclusion_file` (see above)) . Should be a list of +`welcome_file` and `conclusion_file` (see above)) . Should be a list of file paths, relative to the directory where `construct.yaml` is. In Windows, these files will be copied into a temporary folder, the NSIS `$PLUGINSDIR`, during install process (Windows only). diff --git a/constructor/header.sh b/constructor/header.sh index ed88fae5a..fa938c3ae 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -21,6 +21,24 @@ if ! echo "$0" | grep '\.sh$' > /dev/null; then return 1 fi +#if osx and min_osx_version +min_osx_version="__MIN_OSX_VERSION__" +system_osx_version=$(SYSTEM_VERSION_COMPAT=0 sw_vers -productVersion) +if (( $(printf "%02d%02d%02d" ${system_osx_version//./ }) < $(printf "%02d%02d%02d" ${min_osx_version//./ }) )); then + echo "Installer requires macOS >=${min_osx_version}, but system has ${system_osx_version}." + exit 1 +fi +#endif +#if linux and min_glibc_version +min_glibc_version="__MIN_GLIBC_VERSION__" +# ldd reports glibc in the last field of the first line +system_glibc_version=$(ldd --version | awk 'NR==1{print $NF}') +if (( $(printf "%02d%02d%02d" ${system_glibc_version//./ }) < $(printf "%02d%02d%02d" ${min_glibc_version//./ }) )); then + echo "Installer requires GLIBC >=${min_glibc_version}, but system has ${system_glibc_version}." + exit 1 +fi +#endif + # Export variables to make installer metadata available to pre/post install scripts # NOTE: If more vars are added, make sure to update the examples/scripts tests too diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index a514e351c..a48a85d5c 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -10,7 +10,7 @@ from tempfile import NamedTemporaryFile from . import preconda -from .conda_interface import MatchSpec, conda_context +from .conda_interface import conda_context from .construct import ns_platform, parse from .imaging import write_images from .utils import ( @@ -19,6 +19,7 @@ explained_check_call, fill_template, get_final_channels, + parse_virtual_specs, preprocess, rm_rf, shortcuts_flags, @@ -191,27 +192,7 @@ def modify_xml(xml_path, info): # -- __osx virtual package checks -- # # Reference: https://developer.apple.com/library/archive/documentation/DeveloperTools/Reference/DistributionDefinitionRef/Chapters/Distribution_XML_Ref.html # noqa - osx_versions = {} - for spec in info.get("virtual_specs", ()): - spec = MatchSpec(spec) - if spec.name != "__osx": - continue - if not spec.version: - continue - if "|" in spec.version.spec_str: - raise ValueError("Can't process `|`-joined versions. Only `,` is allowed.") - versions = spec.version.tup if "," in spec.version.spec_str else (spec.version,) - for version in versions: - operator = version.operator_func.__name__ - if operator == "ge": - osx_versions["min"] = str(version.matcher_vo) - elif operator == "lt": - osx_versions["before"] = str(version.matcher_vo) - else: - raise ValueError( - f"Invalid version operator for {spec}. Only `<` or `>=` are supported." - ) - + osx_versions = parse_virtual_specs(info)["__osx"] if osx_versions: if "min" not in osx_versions: raise ValueError("Specifying __osx requires a lower bound with `>=`") diff --git a/constructor/shar.py b/constructor/shar.py index 9941967bc..0425f3661 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -24,6 +24,7 @@ fill_template, get_final_channels, hash_files, + parse_virtual_specs, preprocess, read_ascii_only, shortcuts_flags, @@ -98,6 +99,12 @@ def get_header(conda_exec, tarball, info): if has_license: replace['LICENSE'] = read_ascii_only(info['license_file']) + virtual_specs = parse_virtual_specs(info) + if min_osx_version := virtual_specs["__osx"].get("min"): + replace['MIN_OSX_VERSION'] = ppd['min_osx_version'] = min_osx_version + if min_glibc_version := virtual_specs["__glibc"].get("min"): + replace['MIN_GLIBC_VERSION'] = ppd['min_glibc_version'] = min_glibc_version + data = read_header_template() data = preprocess(data, ppd) custom_variables = info.get('script_env_variables', {}) diff --git a/constructor/utils.py b/constructor/utils.py index 90c4b8cea..c432d25c3 100644 --- a/constructor/utils.py +++ b/constructor/utils.py @@ -282,3 +282,30 @@ def check_required_env_vars(env_vars): raise RuntimeError( f"Missing required environment variables {', '.join(missing_vars)}." ) + + +def parse_virtual_specs(info) -> dict: + from .conda_interface import MatchSpec # prevent circular import + + specs = {"osx": {}, "glibc": {}} + for spec in info.get("virtual_specs", ()): + spec = MatchSpec(spec) + if spec.name not in ("__osx", "__glibc"): + continue + if not spec.version: + continue + if "|" in spec.version.spec_str: + raise ValueError("Can't process `|`-joined versions. Only `,` is allowed.") + versions = spec.version.tup if "," in spec.version.spec_str else (spec.version,) + for version in versions: + operator = version.operator_func.__name__ + if operator == "ge": + specs[spec.name]["min"] = str(version.matcher_vo) + elif operator == "lt" and spec.name == "__osx": + specs[spec.name]["before"] = str(version.matcher_vo) + else: + raise ValueError( + f"Invalid version operator for {spec}. " + "__osx only supports `<` or `>=`; __glibc only supports `>=`." + ) + return specs diff --git a/docs/source/construct-yaml.md b/docs/source/construct-yaml.md index d15f5484b..67114e00d 100644 --- a/docs/source/construct-yaml.md +++ b/docs/source/construct-yaml.md @@ -121,8 +121,9 @@ _type:_ list
A list of virtual packages that must be satisfied at install time. Virtual packages must start with `__`. For example, `__osx>=11` or `__glibc>=2.24`. These specs are dry-run solved offline by the bundled `--conda-exe` binary. -In PKG installers, `__osx` specs can be checked natively without the solver -being involved as long as only `>=`, `<` or `,` are used. +In SH installers, `__glibc>=x.y` and `__osx>=x.y` specs can be checked with +Bash only. In PKG installers, `__osx` specs can be checked natively without +the solver being involved as long as only `>=`, `<` or `,` are used. ### `exclude` @@ -821,7 +822,7 @@ _required:_ no
_type:_ list
Temporary files that could be referenced in the installation process (i.e. customized - `welcome_file` and `conclusion_file` (see above)) . Should be a list of +`welcome_file` and `conclusion_file` (see above)) . Should be a list of file paths, relative to the directory where `construct.yaml` is. In Windows, these files will be copied into a temporary folder, the NSIS `$PLUGINSDIR`, during install process (Windows only). diff --git a/examples/virtual_specs/construct.yaml b/examples/virtual_specs/construct.yaml index f387ee2fc..f1d28b747 100644 --- a/examples/virtual_specs/construct.yaml +++ b/examples/virtual_specs/construct.yaml @@ -11,9 +11,9 @@ specs: - ca-certificates virtual_specs: - - __osx>=5.1,<9 # [osx] - - __glibc>=20 # [linux] - - __win<0 # [win] + - __osx>=5.1,<9 # [osx] + - __glibc>=20 # [linux] + - __win<0 # [win] initialize_by_default: false register_python: false From ab606424e749d3265588ce7b101c67278e17c05d Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 25 Jul 2024 12:34:22 +0200 Subject: [PATCH 23/34] fix tests --- tests/test_header.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test_header.py b/tests/test_header.py index b72a8d3ef..539d01e0a 100644 --- a/tests/test_header.py +++ b/tests/test_header.py @@ -57,6 +57,8 @@ def test_linux_template_processing(): enable_shortcuts, check_path_spaces, arch, + min_glibc_version, + min_osx_version, ) in itertools.product( [False, True], [False, True], @@ -72,6 +74,8 @@ def test_linux_template_processing(): [False, True], [False, True], ["x86", "x86_64", " ppc64le", "s390x", "aarch64"], + [None, "2.17"], + [None, "10.13"], ): params = { "has_license": has_license, @@ -93,12 +97,14 @@ def test_linux_template_processing(): "initialize_by_default": initialize_by_default, "enable_shortcuts": enable_shortcuts, "check_path_spaces": check_path_spaces, + "min_glibc_version": min_glibc_version, + "min_osx_version": min_osx_version, } processed = preprocess(template, params) for template_string in ["#if", "#else", "#endif"]: if template_string in processed: errors.append( - f"Found '{template_string}' after " f"processing header.sh with '{params}'." + f"Found '{template_string}' after processing header.sh with '{params}'." ) assert not errors @@ -156,6 +162,8 @@ def test_osxpkg_scripts_shellcheck(arch, check_path_spaces, script): @pytest.mark.parametrize("arch", ["x86_64", "aarch64"]) @pytest.mark.parametrize("check_path_spaces", [True]) @pytest.mark.parametrize("enable_shortcuts", ["true"]) +@pytest.mark.parametrize("min_glibc_version", ["2.17"]) +@pytest.mark.parametrize("min_osx_version", ["10.13"]) def test_template_shellcheck( osx, arch, @@ -171,6 +179,8 @@ def test_template_shellcheck( direct_execute_post_install, check_path_spaces, enable_shortcuts, + min_glibc_version, + min_osx_version, ): template = read_header_template() processed = preprocess( @@ -195,9 +205,12 @@ def test_template_shellcheck( "initialize_by_default": initialize_by_default, "check_path_spaces": check_path_spaces, "enable_shortcuts": enable_shortcuts, + "min_glibc_version": min_glibc_version, + "min_osx_version": min_osx_version, }, ) findings, returncode = run_shellcheck(processed) + print(*findings, sep="\n") assert findings == [] assert returncode == 0 From 8b9146cabb80ddc933081aa3345045ea91e59e33 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 25 Jul 2024 12:34:29 +0200 Subject: [PATCH 24/34] make it posix --- constructor/header.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/constructor/header.sh b/constructor/header.sh index fa938c3ae..3b40232ea 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -24,7 +24,11 @@ fi #if osx and min_osx_version min_osx_version="__MIN_OSX_VERSION__" system_osx_version=$(SYSTEM_VERSION_COMPAT=0 sw_vers -productVersion) -if (( $(printf "%02d%02d%02d" ${system_osx_version//./ }) < $(printf "%02d%02d%02d" ${min_osx_version//./ }) )); then +# shellcheck disable=SC2183 disable=SC2046 +int_min_osx_version="$(printf "%02d%02d%02d" $(echo "$min_osx_version" | sed 's/\./ /g'))" +# shellcheck disable=SC2183 disable=SC2046 +int_system_osx_version="$(printf "%02d%02d%02d" $(echo "$system_osx_version" | sed 's/\./ /g'))" +if [ "$int_system_osx_version" -lt "$int_min_osx_version" ]; then echo "Installer requires macOS >=${min_osx_version}, but system has ${system_osx_version}." exit 1 fi @@ -33,7 +37,11 @@ fi min_glibc_version="__MIN_GLIBC_VERSION__" # ldd reports glibc in the last field of the first line system_glibc_version=$(ldd --version | awk 'NR==1{print $NF}') -if (( $(printf "%02d%02d%02d" ${system_glibc_version//./ }) < $(printf "%02d%02d%02d" ${min_glibc_version//./ }) )); then +# shellcheck disable=SC2183 disable=SC2046 +int_min_glibc_version="$(printf "%02d%02d%02d" $(echo "$min_glibc_version" | sed 's/\./ /g'))" +# shellcheck disable=SC2183 disable=SC2046 +int_system_glibc_version="$(printf "%02d%02d%02d" $(echo "$system_glibc_version" | sed 's/\./ /g'))" +if [ "$int_system_glibc_version" -lt "$int_min_glibc_version" ]; then echo "Installer requires GLIBC >=${min_glibc_version}, but system has ${system_glibc_version}." exit 1 fi From 553b41ab35a0eacdaa48d03b68985cbaf7b42c95 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 25 Jul 2024 12:35:45 +0200 Subject: [PATCH 25/34] pre-commit --- constructor/header.sh | 2 +- constructor/shar.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/constructor/header.sh b/constructor/header.sh index 3b40232ea..753bb1202 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -25,7 +25,7 @@ fi min_osx_version="__MIN_OSX_VERSION__" system_osx_version=$(SYSTEM_VERSION_COMPAT=0 sw_vers -productVersion) # shellcheck disable=SC2183 disable=SC2046 -int_min_osx_version="$(printf "%02d%02d%02d" $(echo "$min_osx_version" | sed 's/\./ /g'))" +int_min_osx_version="$(printf "%02d%02d%02d" $(echo "$min_osx_version" | sed 's/\./ /g'))" # shellcheck disable=SC2183 disable=SC2046 int_system_osx_version="$(printf "%02d%02d%02d" $(echo "$system_osx_version" | sed 's/\./ /g'))" if [ "$int_system_osx_version" -lt "$int_min_osx_version" ]; then diff --git a/constructor/shar.py b/constructor/shar.py index 0425f3661..7f238996c 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -99,12 +99,12 @@ def get_header(conda_exec, tarball, info): if has_license: replace['LICENSE'] = read_ascii_only(info['license_file']) - virtual_specs = parse_virtual_specs(info) + virtual_specs = parse_virtual_specs(info) if min_osx_version := virtual_specs["__osx"].get("min"): replace['MIN_OSX_VERSION'] = ppd['min_osx_version'] = min_osx_version if min_glibc_version := virtual_specs["__glibc"].get("min"): replace['MIN_GLIBC_VERSION'] = ppd['min_glibc_version'] = min_glibc_version - + data = read_header_template() data = preprocess(data, ppd) custom_variables = info.get('script_env_variables', {}) From 86b958d7871ce7ed5f249008bfa838da45d7b262 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 25 Jul 2024 12:40:03 +0200 Subject: [PATCH 26/34] adjust expected output in test --- tests/test_examples.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index e3cf065b3..238c9f437 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -704,9 +704,10 @@ def test_virtual_specs(tmp_path, request): # The GUI does provide a better message with the min version and so on # but on the CLI we fail with this one instead msg = "Cannot install on volume" - elif CONDA_EXE == "micromamba": - msg = "is missing on the system" - else: - msg = "PackagesNotFoundError" + else: + # The shell installer has its own Bash code for __glibc and __osx + # Other virtual specs like __cuda are checked by conda-standalone/micromamba + # and will fail with solver errors like PackagesNotFound etc + msg = "Installer requires" assert process.returncode != 0 assert msg in process.stdout + process.stderr From 6576d5394af583eebff9389d70b5ae4f77ffe6e3 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 25 Jul 2024 12:40:55 +0200 Subject: [PATCH 27/34] pre-commit --- tests/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 238c9f437..8be80a2fa 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -704,7 +704,7 @@ def test_virtual_specs(tmp_path, request): # The GUI does provide a better message with the min version and so on # but on the CLI we fail with this one instead msg = "Cannot install on volume" - else: + else: # The shell installer has its own Bash code for __glibc and __osx # Other virtual specs like __cuda are checked by conda-standalone/micromamba # and will fail with solver errors like PackagesNotFound etc From 28f74d5c66e3d808ceddd05c658f7274fdba5226 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sat, 27 Jul 2024 18:14:48 +0200 Subject: [PATCH 28/34] fix dict access? --- constructor/shar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/constructor/shar.py b/constructor/shar.py index 7f238996c..1bf16f572 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -100,9 +100,9 @@ def get_header(conda_exec, tarball, info): replace['LICENSE'] = read_ascii_only(info['license_file']) virtual_specs = parse_virtual_specs(info) - if min_osx_version := virtual_specs["__osx"].get("min"): + if min_osx_version := virtual_specs.get("__osx", {}).get("min"): replace['MIN_OSX_VERSION'] = ppd['min_osx_version'] = min_osx_version - if min_glibc_version := virtual_specs["__glibc"].get("min"): + if min_glibc_version := virtual_specs.get("__glibc", {}).get("min"): replace['MIN_GLIBC_VERSION'] = ppd['min_glibc_version'] = min_glibc_version data = read_header_template() From a59d2c52e265890fae99171ffc60368398b1bf5a Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sat, 27 Jul 2024 18:26:10 +0200 Subject: [PATCH 29/34] always provide a value even if falsey --- constructor/osxpkg.py | 2 +- constructor/shar.py | 8 ++++---- constructor/utils.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index a48a85d5c..d49b2ad02 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -192,7 +192,7 @@ def modify_xml(xml_path, info): # -- __osx virtual package checks -- # # Reference: https://developer.apple.com/library/archive/documentation/DeveloperTools/Reference/DistributionDefinitionRef/Chapters/Distribution_XML_Ref.html # noqa - osx_versions = parse_virtual_specs(info)["__osx"] + osx_versions = parse_virtual_specs(info).get("__osx", {}) if osx_versions: if "min" not in osx_versions: raise ValueError("Specifying __osx requires a lower bound with `>=`") diff --git a/constructor/shar.py b/constructor/shar.py index 1bf16f572..322077d87 100644 --- a/constructor/shar.py +++ b/constructor/shar.py @@ -100,10 +100,10 @@ def get_header(conda_exec, tarball, info): replace['LICENSE'] = read_ascii_only(info['license_file']) virtual_specs = parse_virtual_specs(info) - if min_osx_version := virtual_specs.get("__osx", {}).get("min"): - replace['MIN_OSX_VERSION'] = ppd['min_osx_version'] = min_osx_version - if min_glibc_version := virtual_specs.get("__glibc", {}).get("min"): - replace['MIN_GLIBC_VERSION'] = ppd['min_glibc_version'] = min_glibc_version + min_osx_version = virtual_specs.get("__osx", {}).get("min") or "" + replace['MIN_OSX_VERSION'] = ppd['min_osx_version'] = min_osx_version + min_glibc_version = virtual_specs.get("__glibc", {}).get("min") or "" + replace['MIN_GLIBC_VERSION'] = ppd['min_glibc_version'] = min_glibc_version data = read_header_template() data = preprocess(data, ppd) diff --git a/constructor/utils.py b/constructor/utils.py index c432d25c3..7bdf1ec5c 100644 --- a/constructor/utils.py +++ b/constructor/utils.py @@ -287,7 +287,7 @@ def check_required_env_vars(env_vars): def parse_virtual_specs(info) -> dict: from .conda_interface import MatchSpec # prevent circular import - specs = {"osx": {}, "glibc": {}} + specs = {"__osx": {}, "__glibc": {}} for spec in info.get("virtual_specs", ()): spec = MatchSpec(spec) if spec.name not in ("__osx", "__glibc"): From 858586d4bfca18c256d09219224edd73abd15f0c Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sun, 28 Jul 2024 13:10:59 +0200 Subject: [PATCH 30/34] Fix TypeError --- tests/test_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_examples.py b/tests/test_examples.py index 8be80a2fa..c41e7bb4b 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -224,7 +224,7 @@ def _sentinel_file_checks(example_path, install_dir): if (example_path / script).exists() and not (install_dir / sentinel).exists(): raise AssertionError( f"Sentinel file for {script_prefix}_install not found! " - f"{install_dir} contents:\n" + "\n".join(sorted(install_dir.iterdir())) + f"{install_dir} contents:\n" + "\n".join(sorted(map(str, install_dir.iterdir()))) ) From 4f46d6c453b719324d71ac5916975b7107b4eeef Mon Sep 17 00:00:00 2001 From: jaimergp Date: Sun, 28 Jul 2024 15:05:34 +0200 Subject: [PATCH 31/34] better test? --- constructor/osxpkg.py | 2 +- examples/virtual_specs/construct.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/constructor/osxpkg.py b/constructor/osxpkg.py index d49b2ad02..faebf490b 100644 --- a/constructor/osxpkg.py +++ b/constructor/osxpkg.py @@ -192,7 +192,7 @@ def modify_xml(xml_path, info): # -- __osx virtual package checks -- # # Reference: https://developer.apple.com/library/archive/documentation/DeveloperTools/Reference/DistributionDefinitionRef/Chapters/Distribution_XML_Ref.html # noqa - osx_versions = parse_virtual_specs(info).get("__osx", {}) + osx_versions = parse_virtual_specs(info).get("__osx") if osx_versions: if "min" not in osx_versions: raise ValueError("Specifying __osx requires a lower bound with `>=`") diff --git a/examples/virtual_specs/construct.yaml b/examples/virtual_specs/construct.yaml index f1d28b747..7b04c0b6a 100644 --- a/examples/virtual_specs/construct.yaml +++ b/examples/virtual_specs/construct.yaml @@ -11,7 +11,7 @@ specs: - ca-certificates virtual_specs: - - __osx>=5.1,<9 # [osx] + - __osx>=30,<31 # [osx] - __glibc>=20 # [linux] - __win<0 # [win] From 3930766aa9e43e72abf2066bad95c8d5d33f3f8d Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 1 Aug 2024 11:04:42 +0200 Subject: [PATCH 32/34] add workaround for musl --- constructor/header.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/constructor/header.sh b/constructor/header.sh index 753bb1202..e9f07066d 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -35,8 +35,16 @@ fi #endif #if linux and min_glibc_version min_glibc_version="__MIN_GLIBC_VERSION__" -# ldd reports glibc in the last field of the first line -system_glibc_version=$(ldd --version | awk 'NR==1{print $NF}') +case "$(ldd --version 2>&1)" in + *musl*) + # musl ldd will report musl version; call ld.so directly + system_glibc_version=$($(ls -1 /lib64/ld-linux-*.so* | head -1) --version | awk 'NR==1{ sub(/\.$/, ""); print $NF}') + ;; + *) + # ldd reports glibc in the last field of the first line + system_glibc_version=$(ldd --version | awk 'NR==1{print $NF}') + ;; +esac # shellcheck disable=SC2183 disable=SC2046 int_min_glibc_version="$(printf "%02d%02d%02d" $(echo "$min_glibc_version" | sed 's/\./ /g'))" # shellcheck disable=SC2183 disable=SC2046 From 4c6664f78cf3afd02f9acc6de44dc3d972c68b50 Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 1 Aug 2024 11:10:59 +0200 Subject: [PATCH 33/34] star this too --- constructor/header.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constructor/header.sh b/constructor/header.sh index e9f07066d..2046f8706 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -38,7 +38,7 @@ min_glibc_version="__MIN_GLIBC_VERSION__" case "$(ldd --version 2>&1)" in *musl*) # musl ldd will report musl version; call ld.so directly - system_glibc_version=$($(ls -1 /lib64/ld-linux-*.so* | head -1) --version | awk 'NR==1{ sub(/\.$/, ""); print $NF}') + system_glibc_version=$($(ls -1 /lib*/ld-linux-*.so* | head -1) --version | awk 'NR==1{ sub(/\.$/, ""); print $NF}') ;; *) # ldd reports glibc in the last field of the first line From 471bad48df605aed3027e997dbfe2f9c8c40fdfe Mon Sep 17 00:00:00 2001 From: jaimergp Date: Thu, 1 Aug 2024 11:28:39 +0200 Subject: [PATCH 34/34] use find --- constructor/header.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/constructor/header.sh b/constructor/header.sh index 2046f8706..129d3da95 100644 --- a/constructor/header.sh +++ b/constructor/header.sh @@ -38,7 +38,7 @@ min_glibc_version="__MIN_GLIBC_VERSION__" case "$(ldd --version 2>&1)" in *musl*) # musl ldd will report musl version; call ld.so directly - system_glibc_version=$($(ls -1 /lib*/ld-linux-*.so* | head -1) --version | awk 'NR==1{ sub(/\.$/, ""); print $NF}') + system_glibc_version=$($(find /lib/ /lib64/ -name 'ld-linux-*.so*' 2>/dev/null | head -1) --version | awk 'NR==1{ sub(/\.$/, ""); print $NF}') ;; *) # ldd reports glibc in the last field of the first line