[RELEASE] cucim v24.12 (#799)

## ❄️ Code freeze for `branch-24.12` and v24.12 release ### What does this mean? Only critical/hotfix level issues should be merged into `branch-24.12` until release (merging of this PR). ### What is the purpose of this PR? - Update documentation - Allow testing for the new release - Enable a means to merge `branch-24.12` into `main` for the release --------- Co-authored-by: Ray Douglass <[email protected]> Co-authored-by: Gregory Lee <[email protected]> Co-authored-by: James Lamb <[email protected]>
rapidsai · Dec 11, 2024 · 3f68577 · 3f68577
1 parent f08280c
commit 3f68577
Show file tree

Hide file tree

Showing 23 changed files with 116 additions and 134 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -28,7 +28,7 @@ concurrency:
 jobs:
   cpp-build:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -38,7 +38,7 @@ jobs:
     if: github.ref_type == 'branch'
     needs: [python-build]
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12
     with:
       arch: "amd64"
       branch: ${{ inputs.branch }}
@@ -51,7 +51,7 @@ jobs:
   python-build:
     needs: [cpp-build]
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -60,15 +60,15 @@ jobs:
   upload-conda:
     needs: [cpp-build, python-build]
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/conda-upload-packages.yaml@branch-24.12
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
   wheel-build:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}
@@ -78,7 +78,7 @@ jobs:
   wheel-publish:
     needs: wheel-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12
     with:
       build_type: ${{ inputs.build_type || 'branch' }}
       branch: ${{ inputs.branch }}

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -20,32 +20,32 @@ jobs:
       - wheel-build
       - wheel-tests
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12
   checks:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12
   conda-cpp-build:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12
     with:
       build_type: pull-request
   conda-python-build:
     needs: conda-cpp-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12
     with:
       build_type: pull-request
   conda-python-tests:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12
     with:
       build_type: pull-request
   docs-build:
     needs: conda-python-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12
     with:
       build_type: pull-request
       node_type: "gpu-v100-latest-1"
@@ -55,14 +55,14 @@ jobs:
   wheel-build:
     needs: checks
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12
     with:
       build_type: pull-request
       script: ci/build_wheel.sh
   wheel-tests:
     needs: wheel-build
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12
     with:
       build_type: pull-request
       script: ci/test_wheel.sh
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -16,15 +16,15 @@ on:
 jobs:
   conda-python-tests:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
   wheel-tests:
     secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10
+    uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12
     with:
       build_type: nightly
       branch: ${{ inputs.branch }}

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -34,7 +34,7 @@ repos:
       - id: verify-alpha-spec
         args: ["--fix", "--mode=release"]
   - repo: https://github.com/rapidsai/dependency-file-generator
-    rev: v1.13.11
+    rev: v1.16.0
     hooks:
         - id: rapids-dependency-file-generator
           args: ["--clean"]
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,17 @@
+# cucim 24.12.00 (11 Dec 2024)
+
+## 🚨 Breaking Changes
+
+- Deprecations: carry out removals scheduled for release 24.12 ([#786](https://github.com/rapidsai/cucim/pull/786)) [@grlee77](https://github.com/grlee77)
+
+## 🛠️ Improvements
+
+- allow path conflicts in conda builds ([#801](https://github.com/rapidsai/cucim/pull/801)) [@jameslamb](https://github.com/jameslamb)
+- enforce wheel size limits, README formatting in CI ([#797](https://github.com/rapidsai/cucim/pull/797)) [@jameslamb](https://github.com/jameslamb)
+- build wheels without build isolation, print sccache stats in builds ([#792](https://github.com/rapidsai/cucim/pull/792)) [@jameslamb](https://github.com/jameslamb)
+- make conda installs in CI stricter ([#791](https://github.com/rapidsai/cucim/pull/791)) [@jameslamb](https://github.com/jameslamb)
+- Deprecations: carry out removals scheduled for release 24.12 ([#786](https://github.com/rapidsai/cucim/pull/786)) [@grlee77](https://github.com/grlee77)
+
 # cucim 24.10.00 (9 Oct 2024)
 
 ## 🐛 Bug Fixes

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-24.10.00
+24.12.00
diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh
@@ -15,8 +15,14 @@ rapids-print-env
 
 rapids-logger "Begin cpp build"
 
-conda config --set path_conflict prevent
+# this can be set back to 'prevent' once the xorg-* migrations are completed
+# ref: https://github.com/rapidsai/cucim/issues/800#issuecomment-2529593457
+conda config --set path_conflict warn
+
+sccache --zero-stats
 
 RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcucim
 
+sccache --show-adv-stats
+
 rapids-upload-conda-to-s3 cpp
diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -2,11 +2,11 @@
 
 set -euo pipefail
 
-export RAPIDS_VERSION_NUMBER="$(rapids-version-major-minor)"
-
 rapids-logger "Create test conda environment"
 . /opt/conda/etc/profile.d/conda.sh
 
+RAPIDS_VERSION="$(rapids-version)"
+
 rapids-dependency-file-generator \
     --output conda \
     --file-key docs \
@@ -24,7 +24,8 @@ PYTHON_CHANNEL=$(rapids-download-conda-from-s3 python)
 rapids-mamba-retry install \
     --channel "${CPP_CHANNEL}" \
     --channel "${PYTHON_CHANNEL}" \
-    cucim libcucim
+    "cucim=${RAPIDS_VERSION}" \
+    "libcucim=${RAPIDS_VERSION}"
 
 export RAPIDS_DOCS_DIR="$(mktemp -d)"
 
@@ -35,4 +36,4 @@ mkdir -p "${RAPIDS_DOCS_DIR}/cucim/"html
 mv _html/* "${RAPIDS_DOCS_DIR}/cucim/html"
 popd
 
-rapids-upload-docs
+RAPIDS_VERSION_NUMBER="$(rapids-version-major-minor)" rapids-upload-docs
diff --git a/ci/build_python.sh b/ci/build_python.sh
@@ -16,15 +16,22 @@ rapids-print-env
 rapids-generate-version > ./VERSION
 
 rapids-logger "Begin py build"
-conda config --set path_conflict prevent
+
+# this can be set back to 'prevent' once the xorg-* migrations are completed
+# ref: https://github.com/rapidsai/cucim/issues/800#issuecomment-2529593457
+conda config --set path_conflict warn
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
+sccache --zero-stats
+
 # TODO: Remove `--no-test` flag once importing on a CPU
 # node works correctly
 RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   conda/recipes/cucim
 
+sccache --show-adv-stats
+
 rapids-upload-conda-to-s3 python
diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -5,7 +5,6 @@ set -euo pipefail
 
 package_name="cucim"
 package_dir="python/cucim"
-package_src_dir="${package_dir}/src/${package_name}"
 
 CMAKE_BUILD_TYPE="release"
 
@@ -16,24 +15,47 @@ rapids-generate-version > ./VERSION
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
+rapids-logger "Generating build requirements"
 
-# Install pip build dependencies (not yet using pyproject.toml)
 rapids-dependency-file-generator \
-  --file-key "py_build" \
-  --output "requirements" \
-  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee build_requirements.txt
-pip install -r build_requirements.txt
+  --output requirements \
+  --file-key "py_build_${package_name}" \
+  --file-key "py_rapids_build_${package_name}" \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" \
+| tee /tmp/requirements-build.txt
+
+rapids-logger "Installing build requirements"
+python -m pip install \
+    -v \
+    --prefer-binary \
+    -r /tmp/requirements-build.txt
+
+sccache --zero-stats
 
 # First build the C++ lib using CMake via the run script
 ./run build_local all ${CMAKE_BUILD_TYPE}
 
+sccache --show-adv-stats
+
 cd "${package_dir}"
 
-python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
+sccache --zero-stats
+
+rapids-logger "Building '${package_name}' wheel"
+python -m pip wheel \
+    -w dist \
+    -v \
+    --no-build-isolation \
+    --no-deps \
+    --disable-pip-version-check \
+    .
+
+sccache --show-adv-stats
 
 mkdir -p final_dist
 python -m auditwheel repair -w final_dist dist/*
 ls -1 final_dist | grep -vqz 'none'
 
+../../ci/validate_wheel.sh final_dist
+
 RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist
diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+wheel_dir_relative_path=$1
+
+rapids-logger "validate packages with 'pydistcheck'"
+
+pydistcheck \
+    --inspect \
+    "$(echo ${wheel_dir_relative_path}/*.whl)"
+
+rapids-logger "validate packages with 'twine'"
+
+twine check \
+    --strict \
+    "$(echo ${wheel_dir_relative_path}/*.whl)"
diff --git a/cucim.code-workspace b/cucim.code-workspace
@@ -33,7 +33,7 @@
 					"CUCIM_TESTDATA_FOLDER": "${workspaceDirectory}/test_data",
 					// Add cuslide plugin's library path to LD_LIBRARY_PATH
 					"LD_LIBRARY_PATH": "${workspaceDirectory}/build-debug/lib:${workspaceDirectory}/cpp/plugins/cucim.kit.cuslide/build-debug/lib:${workspaceDirectory}/temp/cuda/lib64:${os_env:LD_LIBRARY_PATH}",
-					"CUCIM_TEST_PLUGIN_PATH": "cucim.kit.cuslide@24.10.00.so"
+					"CUCIM_TEST_PLUGIN_PATH": "cucim.kit.cuslide@24.12.00.so"
 				},
 				"cwd": "${workspaceDirectory}",
 				"catch2": {
@@ -226,7 +226,7 @@
 					},
 					{
 						"name": "CUCIM_TEST_PLUGIN_PATH",
-						"value": "cucim.kit.cuslide@24.10.00.so"
+						"value": "cucim.kit.cuslide@24.12.00.so"
 					}
 				],
 				"console": "externalTerminal",
@@ -254,7 +254,7 @@
 					},
 					{
 						"name": "CUCIM_TEST_PLUGIN_PATH",
-						"value": "cucim.kit.cuslide@24.10.00.so"
+						"value": "cucim.kit.cuslide@24.12.00.so"
 					}
 				],
 				"console": "externalTerminal",
@@ -286,7 +286,7 @@
 					},
 					{
 						"name": "CUCIM_TEST_PLUGIN_PATH",
-						"value": "cucim.kit.cuslide@24.10.00.so"
+						"value": "cucim.kit.cuslide@24.12.00.so"
 					}
 				],
 				"console": "externalTerminal",

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -31,14 +31,14 @@ files:
       - cuda_version
       - docs
       - py_version
-  py_build:
+  py_build_cucim:
     output: pyproject
     pyproject_dir: python/cucim
     extras:
       table: build-system
     includes:
       - rapids_build_setuptools
-  py_rapids_build:
+  py_rapids_build_cucim:
     output: pyproject
     pyproject_dir: python/cucim
     extras:

diff --git a/python/cucim/pyproject.toml b/python/cucim/pyproject.toml
@@ -115,6 +115,14 @@ where = ["src"]
 [tool.setuptools.package-data]
 mypkg = ["*.pyi", "*.h", "*.cu", "VERSION"]
 
+[tool.pydistcheck]
+select = [
+    "distro-too-large-compressed",
+]
+
+# PyPI limit is 100 MiB, fail CI before we get too close to that
+max_allowed_size_compressed = '75M'
+
 [tool.pytest.ini_options]
 # If a pytest section is found in one of the possible config files
 # (pytest.ini, tox.ini or setup.cfg), then pytest will not look for any others,

diff --git a/python/cucim/src/cucim/skimage/__init__.py b/python/cucim/src/cucim/skimage/__init__.py
@@ -80,4 +80,3 @@ def __dir__():
     img_as_ubyte,
     img_as_uint,
 )
-from .util.lookfor import lookfor