From 3a34b1ab7482071ec5a937c29a4b2e219427f973 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 22 Sep 2023 17:55:46 -0500 Subject: [PATCH 01/14] Use conda.base.constants.KNOWN_SUBDIRS for setting up selectors --- conda_build/metadata.py | 23 ++++++++++++++--------- conda_build/utils.py | 22 +++------------------- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 0681bcf90c..4df03b8789 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -27,6 +27,7 @@ find_recipe, get_installed_packages, insert_variant_versions, + DEFAULT_SUBDIRS, ) from .conda_interface import MatchSpec, envs_dirs, md5_file, non_x86_linux_machines @@ -121,25 +122,29 @@ def get_selectors(config: Config) -> dict[str, bool]: # Remember to update the docs of any of this changes plat = config.host_subdir d = dict( - linux=plat.startswith("linux-"), linux32=bool(plat == "linux-32"), linux64=bool(plat == "linux-64"), - emscripten=plat.startswith("emscripten-"), - wasi=plat.startswith("wasi-"), - arm=plat.startswith("linux-arm"), - osx=plat.startswith("osx-"), unix=plat.startswith(("linux-", "osx-", "emscripten-")), - win=plat.startswith("win-"), win32=bool(plat == "win-32"), win64=bool(plat == "win-64"), - x86=plat.endswith(("-32", "-64")), - x86_64=plat.endswith("-64"), - wasm32=bool(plat.endswith("-wasm32")), os=os, environ=os.environ, nomkl=bool(int(os.environ.get("FEATURE_NOMKL", False))), ) + subdirs = [subdir for subdir in DEFAULT_SUBDIRS if subdir != "noarch"] + subdir_oses = set([subdir.split("-")[0] for subdir in DEFAULT_SUBDIRS]) + subdir_archs = set([subdir.split("-")[1] for subdir in DEFAULT_SUBDIRS]) + + for os in subdir_oses: + d[os] = plat.startswith(f"{os}-") + + for arch in subdir_arches: + arch_full = ARCH_MAP.get(arch, arch) + d[arch_full] = plat.endswith(f"-{arch}") + if arch == "32": + d["x86"] = plat.endswith(("-32", "-64")) + defaults = variants.get_default_variant(config) py = config.variant.get("python", defaults["python"]) # there are times when python comes in as a tuple diff --git a/conda_build/utils.py b/conda_build/utils.py index af5678247e..2703257624 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -61,6 +61,8 @@ CONDA_PACKAGE_EXTENSION_V2 = ".conda" CONDA_PACKAGE_EXTENSIONS = (CONDA_PACKAGE_EXTENSION_V2, CONDA_PACKAGE_EXTENSION_V1) +from conda.base.constants import KNOWN_SUBDIRS + import urllib.parse as urlparse import urllib.request as urllib from contextlib import ExitStack # noqa: F401 @@ -104,25 +106,7 @@ mmap_PROT_READ = 0 if on_win else mmap.PROT_READ mmap_PROT_WRITE = 0 if on_win else mmap.PROT_WRITE -DEFAULT_SUBDIRS = { - "emscripten-wasm32", - "wasi-wasm32", - "linux-64", - "linux-32", - "linux-s390x", - "linux-ppc64", - "linux-ppc64le", - "linux-armv6l", - "linux-armv7l", - "linux-aarch64", - "win-64", - "win-32", - "win-arm64", - "osx-64", - "osx-arm64", - "zos-z", - "noarch", -} +DEFAULT_SUBDIRS = set(KNOWN_SUBDIRS) RUN_EXPORTS_TYPES = { "weak", From ee9f64c070311ac5419738c46778d714d1ddeaff Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 22 Sep 2023 22:48:29 -0500 Subject: [PATCH 02/14] os -> subdir_os to avoid conflicts --- conda_build/metadata.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 4df03b8789..9c50b16d60 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -136,8 +136,8 @@ def get_selectors(config: Config) -> dict[str, bool]: subdir_oses = set([subdir.split("-")[0] for subdir in DEFAULT_SUBDIRS]) subdir_archs = set([subdir.split("-")[1] for subdir in DEFAULT_SUBDIRS]) - for os in subdir_oses: - d[os] = plat.startswith(f"{os}-") + for subdir_os in subdir_oses: + d[subdir_os] = plat.startswith(f"{subdir_os}-") for arch in subdir_arches: arch_full = ARCH_MAP.get(arch, arch) From 7a170d9eb26df80b61fcd651206fb3d23b9dd0c2 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 22 Sep 2023 22:53:01 -0500 Subject: [PATCH 03/14] Add news --- news/5009-use-conda-known-subdirs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 news/5009-use-conda-known-subdirs diff --git a/news/5009-use-conda-known-subdirs b/news/5009-use-conda-known-subdirs new file mode 100644 index 0000000000..a9423202f4 --- /dev/null +++ b/news/5009-use-conda-known-subdirs @@ -0,0 +1,24 @@ +### Enhancements + +* Use subdirs known to conda for selector definitions. (#5009) + This allows conda_build to support new architectures with just + a new version of conda. For new OSes, there are more information + needed for conda_build to work properly, including whether the + new OS is a UNIX-like platform, the shared library prefix, and + the binary archive format for the platform. + +### Bug fixes + +* + +### Deprecations + +* + +### Docs + +* + +### Other + +* From 6b93f0e3ec15a1334eaaaabf41e3c29cdf773416 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 22 Sep 2023 22:55:16 -0500 Subject: [PATCH 04/14] Fix typos --- conda_build/metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 9c50b16d60..dc438795b5 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -133,13 +133,13 @@ def get_selectors(config: Config) -> dict[str, bool]: ) subdirs = [subdir for subdir in DEFAULT_SUBDIRS if subdir != "noarch"] - subdir_oses = set([subdir.split("-")[0] for subdir in DEFAULT_SUBDIRS]) - subdir_archs = set([subdir.split("-")[1] for subdir in DEFAULT_SUBDIRS]) + subdir_oses = set([subdir.split("-")[0] for subdir in subdirs]) + subdir_archs = set([subdir.split("-")[1] for subdir in subdirs]) for subdir_os in subdir_oses: d[subdir_os] = plat.startswith(f"{subdir_os}-") - for arch in subdir_arches: + for arch in subdir_archs: arch_full = ARCH_MAP.get(arch, arch) d[arch_full] = plat.endswith(f"-{arch}") if arch == "32": From bc89c7b3e23d2141fa10694c0df8621caa3fcef2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 23 Sep 2023 19:35:51 +0000 Subject: [PATCH 05/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- conda_build/metadata.py | 6 +++--- conda_build/utils.py | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index dc438795b5..5b7e7c843e 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -21,13 +21,13 @@ from conda_build.features import feature_list from conda_build.license_family import ensure_valid_license_family from conda_build.utils import ( + DEFAULT_SUBDIRS, HashableDict, ensure_list, expand_globs, find_recipe, get_installed_packages, insert_variant_versions, - DEFAULT_SUBDIRS, ) from .conda_interface import MatchSpec, envs_dirs, md5_file, non_x86_linux_machines @@ -133,8 +133,8 @@ def get_selectors(config: Config) -> dict[str, bool]: ) subdirs = [subdir for subdir in DEFAULT_SUBDIRS if subdir != "noarch"] - subdir_oses = set([subdir.split("-")[0] for subdir in subdirs]) - subdir_archs = set([subdir.split("-")[1] for subdir in subdirs]) + subdir_oses = {subdir.split("-")[0] for subdir in subdirs} + subdir_archs = {subdir.split("-")[1] for subdir in subdirs} for subdir_os in subdir_oses: d[subdir_os] = plat.startswith(f"{subdir_os}-") diff --git a/conda_build/utils.py b/conda_build/utils.py index 2703257624..06a5c79c6d 100644 --- a/conda_build/utils.py +++ b/conda_build/utils.py @@ -61,14 +61,13 @@ CONDA_PACKAGE_EXTENSION_V2 = ".conda" CONDA_PACKAGE_EXTENSIONS = (CONDA_PACKAGE_EXTENSION_V2, CONDA_PACKAGE_EXTENSION_V1) -from conda.base.constants import KNOWN_SUBDIRS - import urllib.parse as urlparse import urllib.request as urllib from contextlib import ExitStack # noqa: F401 from glob import glob from conda.api import PackageCacheData # noqa +from conda.base.constants import KNOWN_SUBDIRS # NOQA because it is not used in this file. from conda_build.conda_interface import rm_rf as _rm_rf # noqa From 2dbf53c72413070d77be732e5034fa710f4eac56 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 6 Oct 2023 15:19:42 -0500 Subject: [PATCH 06/14] No need to use non_x86_linux_machines --- conda_build/metadata.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 5b7e7c843e..9767632970 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -30,7 +30,7 @@ insert_variant_versions, ) -from .conda_interface import MatchSpec, envs_dirs, md5_file, non_x86_linux_machines +from .conda_interface import MatchSpec, envs_dirs, md5_file try: import yaml @@ -187,9 +187,6 @@ def get_selectors(config: Config) -> dict[str, bool]: d["lua"] = lua d["luajit"] = bool(lua[0] == "2") - for machine in non_x86_linux_machines: - d[machine] = bool(plat.endswith("-%s" % machine)) - for feature, value in feature_list: d[feature] = value d.update(os.environ) From 96f4eb36afa05ea9bbc403a46434c81b344c0ef7 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 6 Oct 2023 15:26:10 -0500 Subject: [PATCH 07/14] update docs --- docs/source/resources/define-metadata.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/source/resources/define-metadata.rst b/docs/source/resources/define-metadata.rst index d314349b2b..c9e1ddd32b 100644 --- a/docs/source/resources/define-metadata.rst +++ b/docs/source/resources/define-metadata.rst @@ -1928,10 +1928,10 @@ variables are booleans. * - osx - True if the platform is macOS. * - arm64 - - True if the platform is macOS and the Python architecture - is arm64. + - True if the platform is either macOS or Windows and the + Python architecture is arm64. * - unix - - True if the platform is either macOS or Linux. + - True if the platform is either macOS or Linux or emscripten. * - win - True if the platform is Windows. * - win32 @@ -1965,6 +1965,11 @@ The use of the Python version selectors, `py27`, `py34`, etc. is discouraged in favor of the more general comparison operators. Additional selectors in this series will not be added to conda-build. +Note that for each subdir with OS and architecture that `conda` supports, +two preprocessing selectors are created for the OS and the architecture separately +except when the architecture is not a valid python expression (`*-32` and `*-64` +in particular). + Because the selector is any valid Python expression, complicated logic is possible: From 78396920c9dd73d320db6d736e728538f65cba9a Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Fri, 6 Oct 2023 15:28:51 -0500 Subject: [PATCH 08/14] Add current platform to subdirs --- conda_build/metadata.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 9767632970..b59d3cd352 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -133,6 +133,10 @@ def get_selectors(config: Config) -> dict[str, bool]: ) subdirs = [subdir for subdir in DEFAULT_SUBDIRS if subdir != "noarch"] + # Add the current platform to the list as well to enable conda-build + # to bootstrap new platforms without a new conda release. + subdirs.append(plat) + subdir_oses = {subdir.split("-")[0] for subdir in subdirs} subdir_archs = {subdir.split("-")[1] for subdir in subdirs} From 13cd9fdd4db5667a42d05e2e561c348aa5f58f7f Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Sat, 7 Oct 2023 12:56:30 -0500 Subject: [PATCH 09/14] Filter out subdirs without dash --- conda_build/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index b59d3cd352..f00731f2eb 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -132,7 +132,7 @@ def get_selectors(config: Config) -> dict[str, bool]: nomkl=bool(int(os.environ.get("FEATURE_NOMKL", False))), ) - subdirs = [subdir for subdir in DEFAULT_SUBDIRS if subdir != "noarch"] + subdirs = [subdir for subdir in DEFAULT_SUBDIRS if "-" not in subdir] # Add the current platform to the list as well to enable conda-build # to bootstrap new platforms without a new conda release. subdirs.append(plat) From 69f7982170199d78570e6f0c1551abe6373e7f87 Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Sat, 7 Oct 2023 13:14:46 -0500 Subject: [PATCH 10/14] fix typo --- conda_build/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index f00731f2eb..ba59ad7a6f 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -132,7 +132,7 @@ def get_selectors(config: Config) -> dict[str, bool]: nomkl=bool(int(os.environ.get("FEATURE_NOMKL", False))), ) - subdirs = [subdir for subdir in DEFAULT_SUBDIRS if "-" not in subdir] + subdirs = [subdir for subdir in DEFAULT_SUBDIRS if "-" in subdir] # Add the current platform to the list as well to enable conda-build # to bootstrap new platforms without a new conda release. subdirs.append(plat) From 1f28151b231dc5ca54eebb8c96c36d28d9e79bdb Mon Sep 17 00:00:00 2001 From: Isuru Fernando Date: Sun, 8 Oct 2023 00:22:18 -0500 Subject: [PATCH 11/14] fix filtering subdirs --- conda_build/metadata.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index ba59ad7a6f..e0f269f0b7 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -132,10 +132,12 @@ def get_selectors(config: Config) -> dict[str, bool]: nomkl=bool(int(os.environ.get("FEATURE_NOMKL", False))), ) - subdirs = [subdir for subdir in DEFAULT_SUBDIRS if "-" in subdir] - # Add the current platform to the list as well to enable conda-build + # Add the current platform to the list of subdirs to enable conda-build # to bootstrap new platforms without a new conda release. - subdirs.append(plat) + subdirs = list(DEFAULT_SUBDIRS) + [plat] + + # filter out noarch and other weird subdirs + subdirs = [subdir for subdir in subdirs if "-" in subdir] subdir_oses = {subdir.split("-")[0] for subdir in subdirs} subdir_archs = {subdir.split("-")[1] for subdir in subdirs} From ef76ab4ed90d5c0dd96030d490b8fe44af524018 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Fri, 13 Oct 2023 14:40:38 -0500 Subject: [PATCH 12/14] Add missing selectors from test --- tests/test_metadata.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 9c2ab7bd30..37319f0de4 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -342,6 +342,7 @@ def test_yamlize_versions(): "armv6l", "armv7l", "emscripten", + "freebsd", "linux", "linux32", "linux64", @@ -358,6 +359,8 @@ def test_yamlize_versions(): "win64", "x86", "x86_64", + "z", + "zos", ) @@ -369,7 +372,7 @@ def test_yamlize_versions(): [ ("emscripten-wasm32", {"unix", "emscripten", "wasm32"}), ("wasi-wasm32", {"wasi", "wasm32"}), - ("freebsd-64", {"x86", "x86_64"}), + ("freebsd-64", {"freebsd", "x86", "x86_64"}), ("linux-32", {"unix", "linux", "linux32", "x86"}), ("linux-64", {"unix", "linux", "linux64", "x86", "x86_64"}), ("linux-aarch64", {"unix", "linux", "aarch64"}), @@ -384,7 +387,7 @@ def test_yamlize_versions(): ("win-32", {"win", "win32", "x86"}), ("win-64", {"win", "win64", "x86", "x86_64"}), ("win-arm64", {"win", "arm64"}), - ("zos-z", {}), + ("zos-z", {"zos", "z"}), ], ) @pytest.mark.parametrize("nomkl", [0, 1]) From 0fac35cf420f84361eae471835e3e385f5e5bb93 Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Fri, 13 Oct 2023 14:41:27 -0500 Subject: [PATCH 13/14] Add back arm selector --- conda_build/metadata.py | 1 + tests/test_metadata.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 76e5e11335..12acfb5bcc 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -124,6 +124,7 @@ def get_selectors(config: Config) -> dict[str, bool]: d = dict( linux32=bool(plat == "linux-32"), linux64=bool(plat == "linux-64"), + arm=plat.startswith(("linux-arm", "osx-arm", "win-arm")), unix=plat.startswith(("linux-", "osx-", "emscripten-")), win32=bool(plat == "win-32"), win64=bool(plat == "win-64"), diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 37319f0de4..a1c987ea71 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -383,10 +383,10 @@ def test_yamlize_versions(): ("linux-riscv64", {"unix", "linux", "riscv64"}), ("linux-s390x", {"unix", "linux", "s390x"}), ("osx-64", {"unix", "osx", "x86", "x86_64"}), - ("osx-arm64", {"unix", "osx", "arm64"}), + ("osx-arm64", {"unix", "osx", "arm", "arm64"}), ("win-32", {"win", "win32", "x86"}), ("win-64", {"win", "win64", "x86", "x86_64"}), - ("win-arm64", {"win", "arm64"}), + ("win-arm64", {"win", "arm", "arm64"}), ("zos-z", {"zos", "z"}), ], ) From f3486686206c57e242789fe51e9e65b21f49ab8f Mon Sep 17 00:00:00 2001 From: Ken Odegard Date: Sat, 14 Oct 2023 18:45:56 -0500 Subject: [PATCH 14/14] Revert osx-arm64 or win-arm64 implying arm selector Co-authored-by: Isuru Fernando --- conda_build/metadata.py | 2 +- tests/test_metadata.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 12acfb5bcc..d2d87912bf 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -124,7 +124,7 @@ def get_selectors(config: Config) -> dict[str, bool]: d = dict( linux32=bool(plat == "linux-32"), linux64=bool(plat == "linux-64"), - arm=plat.startswith(("linux-arm", "osx-arm", "win-arm")), + arm=plat.startswith("linux-arm"), unix=plat.startswith(("linux-", "osx-", "emscripten-")), win32=bool(plat == "win-32"), win64=bool(plat == "win-64"), diff --git a/tests/test_metadata.py b/tests/test_metadata.py index a1c987ea71..37319f0de4 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -383,10 +383,10 @@ def test_yamlize_versions(): ("linux-riscv64", {"unix", "linux", "riscv64"}), ("linux-s390x", {"unix", "linux", "s390x"}), ("osx-64", {"unix", "osx", "x86", "x86_64"}), - ("osx-arm64", {"unix", "osx", "arm", "arm64"}), + ("osx-arm64", {"unix", "osx", "arm64"}), ("win-32", {"win", "win32", "x86"}), ("win-64", {"win", "win64", "x86", "x86_64"}), - ("win-arm64", {"win", "arm", "arm64"}), + ("win-arm64", {"win", "arm64"}), ("zos-z", {"zos", "z"}), ], )