support stdlib() jinja function (#4999)

* support stdlib() jinja function

* Fix typos

* Fix formatting

* Fix func signature

* re-add compiler to jinja_context-namespace

* add test for stdlib jinja

* Fix tests

* Need platform and arch as well.

Due to the order in which cbc.yaml is parsed.

* add documentation for stdlib jinja-function

* minor edits after review

* flesh out explanation of how `stdlib` is intended to work

* reST nits

* avoid "resp." abbreviation, reformulate sentence & reflow paragraph

---------

Co-authored-by: Isuru Fernando <[email protected]>
Co-authored-by: jakirkham <[email protected]>

conda_build/jinja_context.py

@@ -494,34 +494,42 @@ def native_compiler(language, config):
     return compiler
 
 
-def compiler(language, config, permit_undefined_jinja=False):
-    """Support configuration of compilers.  This is somewhat platform specific.
+def _target(language, config, permit_undefined_jinja=False, component="compiler"):
+    """Support configuration of compilers/stdlib.  This is somewhat platform specific.
 
-    Native compilers never list their host - it is always implied.  Generally, they are
+    Native compilers/stdlib never list their host - it is always implied.  Generally, they are
     metapackages, pointing at a package that does specify the host.  These in turn may be
     metapackages, pointing at a package where the host is the same as the target (both being the
     native architecture).
     """
 
-    compiler = native_compiler(language, config)
+    if component == "compiler":
+        package_prefix = native_compiler(language, config)
+    else:
+        package_prefix = language
+
     version = None
     if config.variant:
         target_platform = config.variant.get("target_platform", config.subdir)
-        language_compiler_key = f"{language}_compiler"
-        # fall back to native if language-compiler is not explicitly set in variant
-        compiler = config.variant.get(language_compiler_key, compiler)
-        version = config.variant.get(language_compiler_key + "_version")
+        language_key = f"{language}_{component}"
+        # fall back to native if language-key is not explicitly set in variant
+        package_prefix = config.variant.get(language_key, package_prefix)
+        version = config.variant.get(language_key + "_version")
     else:
         target_platform = config.subdir
 
-    # support cross compilers.  A cross-compiler package will have a name such as
+    # support cross components.  A cross package will have a name such as
     #    gcc_target
     #    gcc_linux-cos6-64
-    compiler = "_".join([compiler, target_platform])
+    package = f"{package_prefix}_{target_platform}"
     if version:
-        compiler = " ".join((compiler, version))
-        compiler = ensure_valid_spec(compiler, warn=False)
-    return compiler
+        package = f"{package} {version}"
+        package = ensure_valid_spec(package, warn=False)
+    return package
+
+
+# ensure we have compiler in namespace
+compiler = partial(_target, component="compiler")
 
 
 def ccache(method, config, permit_undefined_jinja=False):
@@ -788,7 +796,16 @@ def context_processor(
             skip_build_id=skip_build_id,
         ),
         compiler=partial(
-            compiler, config=config, permit_undefined_jinja=permit_undefined_jinja
+            _target,
+            config=config,
+            permit_undefined_jinja=permit_undefined_jinja,
+            component="compiler",
+        ),
+        stdlib=partial(
+            _target,
+            config=config,
+            permit_undefined_jinja=permit_undefined_jinja,
+            component="stdlib",
         ),
         cdt=partial(cdt, config=config, permit_undefined_jinja=permit_undefined_jinja),
         ccache=partial(

docs/source/resources/compiler-tools.rst

@@ -394,6 +394,71 @@ not available. You'd need to create a metapackage ``m2w64-gcc_win-64`` to
 point at the ``m2w64-gcc`` package, which does exist on the msys2 channel on
 `repo.anaconda.com <https://repo.anaconda.com/>`_.
 
+Expressing the relation between compiler and its standard library
+=================================================================
+
+For most languages, certainly for "c" and for "cxx", compiling any given
+program *may* create a run-time dependence on symbols from the respective
+standard library. For example, the standard library for C on linux is generally
+``glibc``, and a core component of your operating system. Conda is not able to
+change or supersede this library (it would be too risky to try to). A similar
+situation exists on MacOS and on Windows.
+
+Compiler packages usually have two ways to deal with this dependence:
+
+* assume the package must be there (like ``glibc`` on linux).
+* always add a run-time requirement on the respective stdlib (e.g. ``libcxx``
+  on MacOS).
+
+However, even if we assume the package must be there, the information about the
+``glibc`` version is still a highly relevant piece of information, which is
+also why it is reflected in the ``__glibc``
+`virtual package <https://docs.conda.io/projects/conda/en/stable/user-guide/tasks/manage-virtual.html>`_.
+
+For example, newer packages may decide over time to increase the lowest version
+of ``glibc`` that they support. We therefore need a way to express this
+dependence in a way that conda will be able to understand, so that (in
+conjunction with the ``__glibc`` virtual package) the environment resolver will
+not consider those packages on machines whose ``glibc`` version is too old.
+
+The way to do this is to use the Jinja2 function ``{{ stdlib('c') }}``, which
+matches ``{{ compiler('c') }}`` in as many ways as possible. Let's start again
+with the ``conda_build_config.yaml``::
+
+    c_stdlib:
+      - sysroot                     # [linux]
+      - macosx_deployment_target    # [osx]
+    c_stdlib_version:
+      - 2.17                        # [linux]
+      - 10.13                       # [osx]
+
+In the recipe we would then use::
+
+    requirements:
+      build:
+        - {{ compiler('c') }}
+        - {{ stdlib('c') }}
+
+This would then express that the resulting package requires ``sysroot ==2.17``
+(corresponds to ``glibc``) on linux and ``macosx_deployment_target ==10.13`` on
+MacOS in the build environment, respectively. How this translates into a
+run-time dependence can be defined in the metadata of the respective conda
+(meta-)package which represents the standard library (i.e. those defined under
+``c_stdlib`` above).
+
+In this example, ``sysroot 2.17`` would generate a run-export on
+``__glibc >=2.17`` and ``macosx_deployment_target 10.13`` would similarly
+generate ``__osx >=10.13``. This way, we enable packages to define their own
+expectations about the standard library in a unified way, and without
+implicitly depending on some global assumption about what the lower version
+on a given platform must be.
+
+In principle, this facility would make it possible to also express the
+dependence on separate stdlib implementations (like ``musl`` instead of
+``glibc``), or to remove the need to assume that a C++ compiler always needs to
+add a run-export on the C++ stdlib -- it could then be left up to packages
+themselves whether they need ``{{ stdlib('cxx') }}`` or not.
+
 Anaconda compilers implicitly add RPATH pointing to the conda environment
 =========================================================================
 

tests/test-recipes/metadata/_stdlib_jinja2/conda_build_config.yaml

@@ -0,0 +1,8 @@
+c_stdlib:                       # [unix]
+  - sysroot                     # [linux]
+  - macosx_deployment_target    # [osx]
+c_stdlib_version:               # [unix]
+  - 2.12                        # [linux64]
+  - 2.17                        # [aarch64 or ppc64le]
+  - 10.13                       # [osx and x86_64]
+  - 11.0                        # [osx and arm64]

tests/test-recipes/metadata/_stdlib_jinja2/meta.yaml

@@ -0,0 +1,9 @@
+package:
+  name: stdlib-test
+  version: 1.0
+
+requirements:
+  host:
+    - {{ stdlib('c') }}
+    # - {{ stdlib('cxx') }}
+    # - {{ stdlib('fortran') }}

tests/test_metadata.py

@@ -223,6 +223,33 @@ def test_compiler_metadata_cross_compiler():
     )
 
 
+@pytest.mark.parametrize(
+    "platform,arch,stdlibs",
+    [
+        ("linux", "64", {"sysroot_linux-64 2.12.*"}),
+        ("linux", "aarch64", {"sysroot_linux-aarch64 2.17.*"}),
+        ("osx", "64", {"macosx_deployment_target_osx-64 10.13.*"}),
+        ("osx", "arm64", {"macosx_deployment_target_osx-arm64 11.0.*"}),
+    ],
+)
+def test_native_stdlib_metadata(
+    platform: str, arch: str, stdlibs: set[str], testing_config
+):
+    testing_config.platform = platform
+    metadata = api.render(
+        os.path.join(metadata_dir, "_stdlib_jinja2"),
+        config=testing_config,
+        variants={"target_platform": f"{platform}-{arch}"},
+        platform=platform,
+        arch=arch,
+        permit_unsatisfiable_variants=True,
+        finalize=False,
+        bypass_env_check=True,
+        python="3.11",  # irrelevant
+    )[0][0]
+    assert stdlibs <= set(metadata.meta["requirements"]["host"])
+
+
 def test_hash_build_id(testing_metadata):
     testing_metadata.config.variant["zlib"] = "1.2"
     testing_metadata.meta["requirements"]["host"] = ["zlib"]