Merge pull request #23926 from jaimergp/enable-conda-libmamba-solver

Bring over latest changes in conda-smithy (mostly conda-libmamba-solver)

.azure-pipelines/azure-pipelines-osx.yml

@@ -20,5 +20,5 @@ jobs:
       ./.scripts/run_osx_build.sh
     displayName: Run OSX build
 
-  - publish: /Users/runner/Mambaforge/conda-bld/osx-64/
+  - publish: /Users/runner/Miniforge3/conda-bld/osx-64/
     artifact: conda_pkgs_osx

.azure-pipelines/azure-pipelines-win.yml

@@ -2,7 +2,7 @@ jobs:
 - job: win_64
   condition: not(eq(variables['Build.SourceBranch'], 'refs/heads/main'))
   pool:
-    vmImage: windows-2019
+    vmImage: windows-2022
   strategy:
     maxParallel: 4
     matrix:
@@ -16,7 +16,7 @@ jobs:
         scriptSource: inline
         script: |
           import urllib.request
-          url = 'https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Windows-x86_64.exe'
+          url = 'https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Windows-x86_64.exe'
           path = r"$(Build.ArtifactStagingDirectory)/Miniforge.exe"
           urllib.request.urlretrieve(url, path)
     - script: |
@@ -26,46 +26,11 @@ jobs:
       displayName: Add conda to PATH
 
     - script: |
-        call activate base
-        mamba.exe install 'python=3.9' 'conda-build>=3.18' 'conda>4.7.12' 'conda-forge-ci-setup=3' 'networkx=2.4' conda-forge-pinning boa  -c conda-forge --strict-channel-priority --yes
-      displayName: Install conda-build
-
-    - script: set PYTHONUNBUFFERED=1
-      displayName: Set PYTHONUNBUFFERED
-
-    # Configure the VM
-    - script: |
-        call activate base
-        setup_conda_rc .\ ".\recipes" .\.ci_support\%CONFIG%.yaml
-      displayName: conda-forge CI setup
-
-    # Configure the VM.
-    - script: |
-        set "CI=azure"
-        call activate base
-        run_conda_forge_build_setup
-      displayName: conda-forge build setup
-
-    # Find the recipes from main in this PR and remove them.
-    - script: |
-        git fetch --force origin main:main
-        cd recipes
-        for /f "tokens=*" %%a in ('git ls-tree --name-only main -- .') do rmdir /s /q %%a && echo Removing recipe: %%a
-        cd ..
-
-    - script: |
-        REM make sure there is a package directory so that artifact publishing works
-        setlocal enableextensions
-        if not exist C:\\bld\\win-64\\ mkdir C:\\bld\\win-64\\
-      displayName: Make artifact dir
-
-    # Special cased version setting some more things!
-    - script: |
-        call activate base
-        git fetch --force origin main:main
-        python .ci_support\build_all.py --arch 64
-
-      displayName: Build recipe
+        call .scripts\run_win_build.bat
+      displayName: Build recipes
+      env:
+        CI: azure
+        CONDA_BLD_PATH: C:\bld
 
     - publish: C:\\bld\\win-64\\
       artifact: conda_pkgs_win

.ci_support/build_all.py

@@ -188,15 +188,23 @@ def check_recipes_in_correct_dir(root_dir, correct_dir):
 
 
 def read_mambabuild(recipes_dir):
+    """
+    Only use mambabuild if all the recipes require so via 
+    'conda_build_tool: mambabuild' in 'recipes/<recipe>/conda-forge.yml'
+    """
     folders = os.listdir(recipes_dir)
-    use_it = True
+    conda_build_tools = []
     for folder in folders:
+        if folder == "example":
+            continue
         cf = os.path.join(recipes_dir, folder, "conda-forge.yml")
         if os.path.exists(cf):
             with open(cf, "r") as f:
                 cfy = yaml.safe_load(f.read())
-            use_it = use_it and cfy.get("build_with_mambabuild", True)
-    return use_it
+            conda_build_tools.append(cfy.get("conda_build_tool", "conda-build"))
+        else:
+            conda_build_tools.append("conda-build")
+    return all([tool == "mambabuild" for tool in conda_build_tools])
 
 
 def use_mambabuild():

.ci_support/requirements.txt

@@ -0,0 +1,6 @@
+conda>=23.7.3
+conda-libmamba-solver>=23.7.0
+conda-build>=3.16
+conda-forge-ci-setup=3.*
+conda-forge-pinning
+networkx=2.4

.scripts/build_steps.sh

@@ -6,26 +6,38 @@
 # benefit from the improvement.
 
 set -xeuo pipefail
+
+export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/staged-recipes}"
+source "${FEEDSTOCK_ROOT}/.scripts/logging_utils.sh"
+
+# This closes the matching `startgroup` on `run_docker_build.sh`
+( endgroup "Start Docker" ) 2> /dev/null
+
+( startgroup "Configuring conda" ) 2> /dev/null
+
 export PYTHONUNBUFFERED=1
+export CI_SUPPORT="/home/conda/staged-recipes-copy/.ci_support"
 
 cat >~/.condarc <<CONDARC
+always_yes: true
 
 channels:
  - conda-forge
 
 conda-build:
-  root-dir: /home/conda/staged-recipes/build_artifacts
+  root-dir: ${FEEDSTOCK_ROOT}/build_artifacts
 
 pkgs_dirs:
-  - /home/conda/staged-recipes/build_artifacts/pkg_cache
+  - ${FEEDSTOCK_ROOT}/build_artifacts/pkg_cache
   - /opt/conda/pkgs
 
 show_channel_urls: true
 
+solver: libmamba
 CONDARC
 
 # Copy the host recipes folder so we don't ever muck with it
-cp -r /home/conda/staged-recipes ~/staged-recipes-copy
+cp -r ${FEEDSTOCK_ROOT} ~/staged-recipes-copy
 
 # Remove any macOS system files
 find ~/staged-recipes-copy/recipes -maxdepth 1 -name ".DS_Store" -delete
@@ -34,20 +46,17 @@ find ~/staged-recipes-copy/recipes -maxdepth 1 -name ".DS_Store" -delete
 echo "Pending recipes."
 ls -la ~/staged-recipes-copy/recipes
 echo "Finding recipes merged in main and removing them from the build."
-pushd /home/conda/staged-recipes/recipes > /dev/null
+pushd "${FEEDSTOCK_ROOT}/recipes" > /dev/null
 if [ "${AZURE}" == "True" ]; then
     git fetch --force origin main:main
 fi
 git ls-tree --name-only main -- . | xargs -I {} sh -c "rm -rf ~/staged-recipes-copy/recipes/{} && echo Removing recipe: {}"
 popd > /dev/null
 
 
-# Make sure build_artifacts is a valid channel
-conda index /home/conda/staged-recipes/build_artifacts
 
-mamba install --yes --quiet "conda>4.7.12" conda-forge-ci-setup=3.* conda-forge-pinning networkx=2.4 "conda-build>=3.16" "boa"
-export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/staged-recipes}"
-export CI_SUPPORT="/home/conda/staged-recipes-copy/.ci_support"
+conda install --quiet --file ${FEEDSTOCK_ROOT}/.ci_support/requirements.txt
+
 setup_conda_rc "${FEEDSTOCK_ROOT}" "/home/conda/staged-recipes-copy/recipes" "${CI_SUPPORT}/${CONFIG}.yaml"
 source run_conda_forge_build_setup
 
@@ -56,6 +65,14 @@ find ~/staged-recipes-copy/recipes -mindepth 2 -maxdepth 2 -type f -name "yum_re
     | xargs -n1 cat | { grep -v -e "^#" -e "^$" || test $? == 1; } | \
     xargs -r /usr/bin/sudo -n yum install -y
 
+# Make sure build_artifacts is a valid channel
+conda index ${FEEDSTOCK_ROOT}/build_artifacts
+
+( endgroup "Configuring conda" ) 2> /dev/null
+
+echo "Building all recipes"
 python ${CI_SUPPORT}/build_all.py
 
-touch "/home/conda/staged-recipes/build_artifacts/conda-forge-build-done"
+( startgroup "Final checks" ) 2> /dev/null
+
+touch "${FEEDSTOCK_ROOT}/build_artifacts/conda-forge-build-done"

.scripts/logging_utils.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# Provide a unified interface for the different logging
+# utilities CI providers offer. If unavailable, provide
+# a compatible fallback (e.g. bare `echo xxxxxx`).
+
+function startgroup {
+    # Start a foldable group of log lines
+    # Pass a single argument, quoted
+    case ${CI:-} in
+        azure )
+            echo "##[group]$1";;
+        travis )
+            echo "$1"
+            echo -en 'travis_fold:start:'"${1// /}"'\\r';;
+        github_actions )
+            echo "::group::$1";;
+        * )
+            echo "$1";;
+    esac
+} 2> /dev/null
+
+function endgroup {
+    # End a foldable group of log lines
+    # Pass a single argument, quoted
+
+    case ${CI:-} in
+        azure )
+            echo "##[endgroup]";;
+        travis )
+            echo -en 'travis_fold:end:'"${1// /}"'\\r';;
+        github_actions )
+            echo "::endgroup::";;
+    esac
+} 2> /dev/null

.scripts/run_docker_build.sh

@@ -2,6 +2,10 @@
 
 # NOTE: This script has been adapted from content generated by github.com/conda-forge/conda-smithy
 
+source .scripts/logging_utils.sh
+
+( startgroup "Configure Docker" ) 2> /dev/null
+
 REPO_ROOT=$(cd "$(dirname "$0")/.."; pwd;)
 ARTIFACTS="$REPO_ROOT/build_artifacts"
 THISDIR="$( cd "$( dirname "$0" )" >/dev/null && pwd )"
@@ -43,6 +47,10 @@ DOCKER_RUN_ARGS="-it"
 if [ "${AZURE}" == "True" ]; then
     DOCKER_RUN_ARGS=""
 fi
+( endgroup "Configure Docker" ) 2> /dev/null
+
+( startgroup "Start Docker" ) 2> /dev/null
+# this group is closed in build_steps.sh
 
 docker pull "${DOCKER_IMAGE}"
 docker run ${DOCKER_RUN_ARGS} \
@@ -59,3 +67,6 @@ docker run ${DOCKER_RUN_ARGS} \
 
 # verify that the end of the script was reached
 test -f "$DONE_CANARY"
+
+# This closes the last group opened in `build_steps.sh`
+( endgroup "Final checks" ) 2> /dev/null

.scripts/run_osx_build.sh

@@ -2,35 +2,42 @@
 
 set -x
 
-echo -e "\n\nInstalling a fresh version of Mambaforge."
-if [[ ${CI} == "travis" ]]; then
-  echo -en 'travis_fold:start:install_mambaforge\\r'
-fi
-MAMBAFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download"
-MAMBAFORGE_FILE="Mambaforge-MacOSX-x86_64.sh"
-curl -L -O "${MAMBAFORGE_URL}/${MAMBAFORGE_FILE}"
-bash $MAMBAFORGE_FILE -b
-if [[ ${CI} == "travis" ]]; then
-  echo -en 'travis_fold:end:install_mambaforge\\r'
-fi
+source .scripts/logging_utils.sh
+
+( startgroup "Ensuring Miniforge" ) 2> /dev/null
+
+MINIFORGE_URL="https://github.com/conda-forge/miniforge/releases/latest/download"
+MINIFORGE_FILE="Miniforge3-MacOSX-x86_64.sh"
+MINIFORGE_ROOT="${MINIFORGE_ROOT:-${HOME}/Miniforge3}"
 
-echo -e "\n\nConfiguring conda."
-if [[ ${CI} == "travis" ]]; then
-  echo -en 'travis_fold:start:configure_conda\\r'
+if [[ -d "${MINIFORGE_ROOT}" ]]; then
+  echo "Miniforge already installed at ${MINIFORGE_ROOT}."
+else
+  echo "Installing Miniforge"
+  curl -L -O "${MINIFORGE_URL}/${MINIFORGE_FILE}"
+  bash $MINIFORGE_FILE -bp "${MINIFORGE_ROOT}"
 fi
 
-source ${HOME}/Mambaforge/etc/profile.d/conda.sh
-conda activate base
+( endgroup "Ensuring Miniforge" ) 2> /dev/null
 
-echo -e "\n\nInstalling conda-forge-ci-setup=3, conda-build and boa."
-mamba install -n base --yes --quiet "conda>4.7.12" conda-forge-ci-setup=3.* conda-forge-pinning networkx=2.4 "conda-build>=3.16" "boa"
+( startgroup "Configuring conda" ) 2> /dev/null
 
+cat >~/.condarc <<CONDARC
+always_yes: true
+show_channel_urls: true
+solver: libmamba
+CONDARC
 
+source "${MINIFORGE_ROOT}/etc/profile.d/conda.sh"
+conda activate base
+
+echo -e "\n\nInstalling conda-forge-ci-setup=3, conda-build."
+conda install --quiet --file .ci_support/requirements.txt
 
 echo -e "\n\nSetting up the condarc and mangling the compiler."
 setup_conda_rc ./ ./recipes ./.ci_support/${CONFIG}.yaml
 if [[ "${CI:-}" != "" ]]; then
-  mangle_compiler ./ ./recipe .ci_support/${CONFIG}.yaml
+  mangle_compiler ./ ./recipes .ci_support/${CONFIG}.yaml
 fi
 
 if [[ "${CI:-}" != "" ]]; then
@@ -44,18 +51,12 @@ fi
 echo -e "\n\nRunning the build setup script."
 source run_conda_forge_build_setup
 
-
-if [[ ${CI} == "travis" ]]; then
-  echo -en 'travis_fold:end:configure_conda\\r'
-fi
-
 set -e
 
 # make sure there is a package directory so that artifact publishing works
-mkdir -p /Users/runner/Mambaforge/conda-bld/osx-64/
+mkdir -p "${MINIFORGE_ROOT}/conda-bld/osx-64/"
 
 # Find the recipes from main in this PR and remove them.
-
 echo ""
 echo "Finding recipes merged in main and removing them from the build."
 pushd ./recipes > /dev/null
@@ -64,5 +65,8 @@ git ls-tree --name-only main -- . | xargs -I {} sh -c "rm -rf {} && echo Removin
 popd > /dev/null
 echo ""
 
+( endgroup "Configuring conda" ) 2> /dev/null
+
 # We just want to build all of the recipes.
+echo "Building all recipes"
 python .ci_support/build_all.py