From a4e21605acfe72a0b3cc125f2db1598c6cf51722 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Wed, 25 Oct 2023 16:13:47 -0400 Subject: [PATCH 01/63] add wheel building scripts and workflow entries --- .github/workflows/build.yaml | 20 +++++++ .github/workflows/pr.yaml | 14 +++++ .github/workflows/test.yaml | 9 +++ ci/build_wheel.sh | 55 +++++++++++++++++++ ci/cmake-omit-benchmarks-examples-tests.patch | 49 +++++++++++++++++ ci/test_wheel.sh | 17 ++++++ ci/wheel_smoke_test.py | 21 +++++++ 7 files changed, 185 insertions(+) create mode 100644 ci/build_wheel.sh create mode 100644 ci/cmake-omit-benchmarks-examples-tests.patch create mode 100644 ci/test_wheel.sh create mode 100644 ci/wheel_smoke_test.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 9e00fdce7..915fd4c4f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,3 +66,23 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + wheel-build: + needs: [cpp-build] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel.sh + wheel-publish: + needs: wheel-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-23.12 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: cucim diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c1c7b60ff..953072d43 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -50,3 +50,17 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" + wheel-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 + with: + build_type: pull-request + script: ci/build_wheel.sh + wheel-tests: + needs: wheel-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12 + with: + build_type: pull-request + script: ci/test_wheel.sh diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9a5e0428a..8ac82418e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,3 +22,12 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + wheel-tests: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-23.12 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/test_wheel.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh new file mode 100644 index 000000000..10700593a --- /dev/null +++ b/ci/build_wheel.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="cucim" +package_dir="python/cucim" + +source rapids-configure-sccache +source rapids-date-string + +# Use gha-tools rapids-pip-wheel-version to generate wheel version then +# update the necessary files +version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +# Patch project metadata files to include the CUDA version suffix and version override. +pyproject_file="${package_dir}/pyproject.toml" + +sed -i "s/^version = .*/version = \"${version_override}\"/g" ${pyproject_file} +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} + +python -m pip install --upgrade pip + +#CMake version in the container is too old, install a new version in the python venv +python -m pip install "cmake>=3.26.4" ninja + +# apply patch to omit building tests and benchmark binaries so libopenslide isn't required +git apply ci/cmake-omit-benchmarks-examples-tests.patch + +# First build the C++ lib using CMake via the run script +./run build_local libcucim release + +# problems: boost-header-only takes a long time to download +# Fails to build any files requiring libopenslide as it isn't on the system + +# Compile the Python bindings +./run build_local cucim release + +# Copy the resulting cucim pybind11 shared library into the Python package src folder +cp -P python/install/lib/* python/cucim/src/cucim/clara/ + +cd "${package_dir}" + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/cmake-omit-benchmarks-examples-tests.patch b/ci/cmake-omit-benchmarks-examples-tests.patch new file mode 100644 index 000000000..f7b244d75 --- /dev/null +++ b/ci/cmake-omit-benchmarks-examples-tests.patch @@ -0,0 +1,49 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 0ab2e17..a3acf3d 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -138,8 +138,8 @@ set(CUCIM_PACKAGE_NAME cucim) # cucim + ################################################################################ + add_subdirectory(cpp) + add_subdirectory(gds) +-add_subdirectory(benchmarks) +-add_subdirectory(examples/cpp) ++# add_subdirectory(benchmarks) ++# add_subdirectory(examples/cpp) + + ################################################################################ + # Write CMakeLists.txt for C++ examples +@@ -155,7 +155,7 @@ configure_file(${CMAKE_CURRENT_LIST_DIR}/examples/cpp/CMakeLists.txt.examples.re + set(INSTALL_TARGETS + ${CUCIM_PACKAGE_NAME} + fmt-header-only +- cucim_benchmarks ++ # cucim_benchmarks + ) + + install(TARGETS ${INSTALL_TARGETS} +diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt +index 67409a6..d2230b3 100644 +--- a/cpp/CMakeLists.txt ++++ b/cpp/CMakeLists.txt +@@ -214,7 +214,7 @@ add_library(${CUCIM_PACKAGE_NAME}::${CUCIM_PACKAGE_NAME}-header-only ALIAS ${CUC + ################################################################################ + # Add tests + ################################################################################ +-add_subdirectory(tests) ++# add_subdirectory(tests) + + ################################################################################# + ## Add bindings +diff --git a/run b/run +index ca4cfd4..dcbc5fd 100755 +--- a/run ++++ b/run +@@ -13,6 +13,7 @@ + # See the License for the specific language governing permissions and + # limitations under the License. + # ++set -x + + init_globals() { + if [ "$0" != "/bin/bash" ]; then diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh new file mode 100644 index 000000000..a608a0eb7 --- /dev/null +++ b/ci/test_wheel.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -eou pipefail + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="cucim_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install $(echo ./dist/cucim*.whl)[test] + +# Run smoke tests for aarch64 pull requests +if [ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]; then + python ./ci/wheel_smoke_test.py +else + python -m pytest ./python/cucim +fi diff --git a/ci/wheel_smoke_test.py b/ci/wheel_smoke_test.py new file mode 100644 index 000000000..82580e80e --- /dev/null +++ b/ci/wheel_smoke_test.py @@ -0,0 +1,21 @@ +import cupy as cp + +import cucim +import cucim.skimage + + +if __name__ == "__main__": + # verify that all top-level modules are available + assert cucim.is_available('clara') + assert cucim.is_available('core') + assert cucim.is_available('skimage') + + # generate a synthetic image and apply a filter + img = cucim.skimage.data.binary_blobs(length=512, n_dim=2) + assert isinstance(img, cp.ndarray) + assert img.dtype.kind == 'b' + assert img.shape == (512, 512) + + eroded = cucim.skimage.morphology.binary_erosion( + img, cp.ones((3, 3), dtype=bool) + ) From 29f5e9d693b09c5635981fa883425d0d4be8353a Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 13:57:24 -0400 Subject: [PATCH 02/63] update pr-builder entries --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 953072d43..d098ff326 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,6 +17,8 @@ jobs: - conda-python-build - conda-python-tests - docs-build + - wheel-build + - wheel-tests secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-23.12 checks: From cdbb0b2b28563d100fce57ad8a5f88dd7859b1ad Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 14:03:50 -0400 Subject: [PATCH 03/63] make wheel build/test scripts executable --- ci/build_wheel.sh | 0 ci/test_wheel.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/build_wheel.sh mode change 100644 => 100755 ci/test_wheel.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh old mode 100644 new mode 100755 diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh old mode 100644 new mode 100755 From 3ec1147750751b65383126c3af42cf260729e5d0 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 14:22:13 -0400 Subject: [PATCH 04/63] call pip directly --- ci/build_wheel.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 10700593a..221b2a420 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -25,14 +25,17 @@ pyproject_file="${package_dir}/pyproject.toml" sed -i "s/^version = .*/version = \"${version_override}\"/g" ${pyproject_file} sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -python -m pip install --upgrade pip +pip install --upgrade pip #CMake version in the container is too old, install a new version in the python venv -python -m pip install "cmake>=3.26.4" ninja +pip install "cmake>=3.26.4" ninja + +echo `which cmake` # apply patch to omit building tests and benchmark binaries so libopenslide isn't required git apply ci/cmake-omit-benchmarks-examples-tests.patch + # First build the C++ lib using CMake via the run script ./run build_local libcucim release From fd29f8e3160b6d8b23c439d862117076e4e1f76f Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 17:33:56 -0400 Subject: [PATCH 05/63] temporarily disable install step of cucim bindings build --- ci/build_wheel.sh | 9 ++++++--- ci/disable-cucim-install-step.patch | 13 +++++++++++++ run | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 ci/disable-cucim-install-step.patch diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 221b2a420..7422b2387 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -6,6 +6,8 @@ set -euo pipefail package_name="cucim" package_dir="python/cucim" +CMAKE_BUILD_TYPE="release" + source rapids-configure-sccache source rapids-date-string @@ -37,16 +39,17 @@ git apply ci/cmake-omit-benchmarks-examples-tests.patch # First build the C++ lib using CMake via the run script -./run build_local libcucim release +./run build_local libcucim ${CMAKE_BUILD_TYPE} # problems: boost-header-only takes a long time to download # Fails to build any files requiring libopenslide as it isn't on the system # Compile the Python bindings -./run build_local cucim release +./run build_local cucim ${CMAKE_BUILD_TYPE} # Copy the resulting cucim pybind11 shared library into the Python package src folder -cp -P python/install/lib/* python/cucim/src/cucim/clara/ +# cp -P python/install/lib/* python/cucim/src/cucim/clara/ +cp -P python/build-${CMAKE_BUILD_TYPE}/lib/cucim/* python/cucim/src/cucim/clara/ cd "${package_dir}" diff --git a/ci/disable-cucim-install-step.patch b/ci/disable-cucim-install-step.patch new file mode 100644 index 000000000..2fc9889d6 --- /dev/null +++ b/ci/disable-cucim-install-step.patch @@ -0,0 +1,13 @@ +diff --git a/run b/run +index ca4cfd4..832c383 100755 +--- a/run ++++ b/run +@@ -370,7 +370,7 @@ build_local_cucim_() { + -DPYTHON_LIBRARY=${python_library} \ + -DPYTHON_INCLUDE_DIR=${python_include_dir} + ${CMAKE_CMD} --build ${build_folder} --config ${build_type_str} --target cucim -- -j $(nproc) +- ${CMAKE_CMD} --build ${build_folder} --config ${build_type_str} --target install -- -j $(nproc) ++ # ${CMAKE_CMD} --build ${build_folder} --config ${build_type_str} --target install -- -j $(nproc) + + popd + } diff --git a/run b/run index b66c2f75a..0a2a04fab 100755 --- a/run +++ b/run @@ -370,7 +370,7 @@ build_local_cucim_() { -DPYTHON_LIBRARY=${python_library} \ -DPYTHON_INCLUDE_DIR=${python_include_dir} ${CMAKE_CMD} --build ${build_folder} --config ${build_type_str} --target cucim -- -j $(nproc) - ${CMAKE_CMD} --build ${build_folder} --config ${build_type_str} --target install -- -j $(nproc) + # ${CMAKE_CMD} --build ${build_folder} --config ${build_type_str} --target install -- -j $(nproc) popd } From f2c6323e712949158d0d83f29a9123c9aeb84bad Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 17:43:51 -0400 Subject: [PATCH 06/63] apply the new patch --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 7422b2387..76916bd27 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -36,7 +36,7 @@ echo `which cmake` # apply patch to omit building tests and benchmark binaries so libopenslide isn't required git apply ci/cmake-omit-benchmarks-examples-tests.patch - +git apply ci/disable-cucim-install-step.patch # First build the C++ lib using CMake via the run script ./run build_local libcucim ${CMAKE_BUILD_TYPE} From c2b7e8226386e31518d5c9816b039793cb3445c8 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 18:26:28 -0400 Subject: [PATCH 07/63] remove patches and install YASM and openslide dev libs via system package manager --- ci/build_wheel.sh | 18 +++++-- ci/cmake-omit-benchmarks-examples-tests.patch | 49 ------------------- ci/disable-cucim-install-step.patch | 13 ----- 3 files changed, 14 insertions(+), 66 deletions(-) delete mode 100644 ci/cmake-omit-benchmarks-examples-tests.patch delete mode 100644 ci/disable-cucim-install-step.patch diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 76916bd27..c69cef2ba 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -34,15 +34,25 @@ pip install "cmake>=3.26.4" ninja echo `which cmake` -# apply patch to omit building tests and benchmark binaries so libopenslide isn't required -git apply ci/cmake-omit-benchmarks-examples-tests.patch -git apply ci/disable-cucim-install-step.patch +# Building the libjpeg-turbo dependency requires YASM +# Also need to install openslide dev libraries on the system +if ! command -v apt &> /dev/null +then + echo "apt package manager not found, attempting to use yum" + yum install yasm openslide-devel -y +else + echo "apt package manager was found" + apt install yasm libopenslide-dev -y +fi # First build the C++ lib using CMake via the run script ./run build_local libcucim ${CMAKE_BUILD_TYPE} +# Build the C++ cuslide and cumed plugins +./run build_local libcuslide ${CMAKE_BUILD_TYPE} +./run build_local libcumed ${CMAKE_BUILD_TYPE} + # problems: boost-header-only takes a long time to download -# Fails to build any files requiring libopenslide as it isn't on the system # Compile the Python bindings ./run build_local cucim ${CMAKE_BUILD_TYPE} diff --git a/ci/cmake-omit-benchmarks-examples-tests.patch b/ci/cmake-omit-benchmarks-examples-tests.patch deleted file mode 100644 index f7b244d75..000000000 --- a/ci/cmake-omit-benchmarks-examples-tests.patch +++ /dev/null @@ -1,49 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 0ab2e17..a3acf3d 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -138,8 +138,8 @@ set(CUCIM_PACKAGE_NAME cucim) # cucim - ################################################################################ - add_subdirectory(cpp) - add_subdirectory(gds) --add_subdirectory(benchmarks) --add_subdirectory(examples/cpp) -+# add_subdirectory(benchmarks) -+# add_subdirectory(examples/cpp) - - ################################################################################ - # Write CMakeLists.txt for C++ examples -@@ -155,7 +155,7 @@ configure_file(${CMAKE_CURRENT_LIST_DIR}/examples/cpp/CMakeLists.txt.examples.re - set(INSTALL_TARGETS - ${CUCIM_PACKAGE_NAME} - fmt-header-only -- cucim_benchmarks -+ # cucim_benchmarks - ) - - install(TARGETS ${INSTALL_TARGETS} -diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt -index 67409a6..d2230b3 100644 ---- a/cpp/CMakeLists.txt -+++ b/cpp/CMakeLists.txt -@@ -214,7 +214,7 @@ add_library(${CUCIM_PACKAGE_NAME}::${CUCIM_PACKAGE_NAME}-header-only ALIAS ${CUC - ################################################################################ - # Add tests - ################################################################################ --add_subdirectory(tests) -+# add_subdirectory(tests) - - ################################################################################# - ## Add bindings -diff --git a/run b/run -index ca4cfd4..dcbc5fd 100755 ---- a/run -+++ b/run -@@ -13,6 +13,7 @@ - # See the License for the specific language governing permissions and - # limitations under the License. - # -+set -x - - init_globals() { - if [ "$0" != "/bin/bash" ]; then diff --git a/ci/disable-cucim-install-step.patch b/ci/disable-cucim-install-step.patch deleted file mode 100644 index 2fc9889d6..000000000 --- a/ci/disable-cucim-install-step.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/run b/run -index ca4cfd4..832c383 100755 ---- a/run -+++ b/run -@@ -370,7 +370,7 @@ build_local_cucim_() { - -DPYTHON_LIBRARY=${python_library} \ - -DPYTHON_INCLUDE_DIR=${python_include_dir} - ${CMAKE_CMD} --build ${build_folder} --config ${build_type_str} --target cucim -- -j $(nproc) -- ${CMAKE_CMD} --build ${build_folder} --config ${build_type_str} --target install -- -j $(nproc) -+ # ${CMAKE_CMD} --build ${build_folder} --config ${build_type_str} --target install -- -j $(nproc) - - popd - } From 6d453ee4921660a6cb173fb5cfa5129af2b9a998 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 18:41:10 -0400 Subject: [PATCH 08/63] add missing apt update call --- ci/build_wheel.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index c69cef2ba..b45318791 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -42,6 +42,7 @@ then yum install yasm openslide-devel -y else echo "apt package manager was found" + apt update apt install yasm libopenslide-dev -y fi From 179482452af6f02b9e79c85d83dcf3f3148e99e9 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 19:20:44 -0400 Subject: [PATCH 09/63] update build_wheel.sh --- ci/build_wheel.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index b45318791..05f684463 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -44,14 +44,25 @@ else echo "apt package manager was found" apt update apt install yasm libopenslide-dev -y + dpkg -L libopenslide-dev + + # temporarily add a sleep here to slow down the aarch64 workers to allow + # x86_64 cases to try to run instead of being cancelled if the aarch64 + # case failed + sleep 10m fi # First build the C++ lib using CMake via the run script ./run build_local libcucim ${CMAKE_BUILD_TYPE} # Build the C++ cuslide and cumed plugins -./run build_local libcuslide ${CMAKE_BUILD_TYPE} -./run build_local libcumed ${CMAKE_BUILD_TYPE} +./run build_local cuslide ${CMAKE_BUILD_TYPE} +cp -P -r cpp/plugins/cucim.kit.cuslide/install/lib/* ./install/lib/ +cp -P -r cpp/plugins/cucim.kit.cuslide/install/bin/* ./install/bin/ + +./run build_local cumed ${CMAKE_BUILD_TYPE} +cp -P -r cpp/plugins/cucim.kit.cumed/install/lib/* ./install/lib/ +cp -P -r cpp/plugins/cucim.kit.cumed/install/bin/* ./install/bin/ # problems: boost-header-only takes a long time to download @@ -59,8 +70,9 @@ fi ./run build_local cucim ${CMAKE_BUILD_TYPE} # Copy the resulting cucim pybind11 shared library into the Python package src folder -# cp -P python/install/lib/* python/cucim/src/cucim/clara/ -cp -P python/build-${CMAKE_BUILD_TYPE}/lib/cucim/* python/cucim/src/cucim/clara/ +cp -P python/install/lib/* python/cucim/src/cucim/clara/ +# also need these files in the clara wheel +cp -P install/lib/* python/cucim/src/cucim/clara/ cd "${package_dir}" From bcb3e19312490506fd435419721510b48cb941c5 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 21:23:51 -0400 Subject: [PATCH 10/63] restore previously commented line --- run | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run b/run index 0a2a04fab..b66c2f75a 100755 --- a/run +++ b/run @@ -370,7 +370,7 @@ build_local_cucim_() { -DPYTHON_LIBRARY=${python_library} \ -DPYTHON_INCLUDE_DIR=${python_include_dir} ${CMAKE_CMD} --build ${build_folder} --config ${build_type_str} --target cucim -- -j $(nproc) - # ${CMAKE_CMD} --build ${build_folder} --config ${build_type_str} --target install -- -j $(nproc) + ${CMAKE_CMD} --build ${build_folder} --config ${build_type_str} --target install -- -j $(nproc) popd } From 6036955d963fce097805c80a5de2b278ea016b90 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 21:35:35 -0400 Subject: [PATCH 11/63] fix copy command (don't add -r and use *.so so cmake subfolder isn't copied) --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 05f684463..dc67dc400 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -72,7 +72,7 @@ cp -P -r cpp/plugins/cucim.kit.cumed/install/bin/* ./install/bin/ # Copy the resulting cucim pybind11 shared library into the Python package src folder cp -P python/install/lib/* python/cucim/src/cucim/clara/ # also need these files in the clara wheel -cp -P install/lib/* python/cucim/src/cucim/clara/ +cp -P install/lib/*.so python/cucim/src/cucim/clara/ cd "${package_dir}" From 94c0e93d9cb716695c448764a5ce568113f44086 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 22:29:07 -0400 Subject: [PATCH 12/63] omit build of cuslide_benchmarks and cuslide_tests on arm64 --- ci/build_wheel.sh | 5 +---- ci/omit-cuslide-benchmarks-and-tests.patch | 25 ++++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 ci/omit-cuslide-benchmarks-and-tests.patch diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index dc67dc400..29aa4d860 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -46,10 +46,7 @@ else apt install yasm libopenslide-dev -y dpkg -L libopenslide-dev - # temporarily add a sleep here to slow down the aarch64 workers to allow - # x86_64 cases to try to run instead of being cancelled if the aarch64 - # case failed - sleep 10m + git apply ci/omit-cuslide-benchmarks-and-tests.patch fi # First build the C++ lib using CMake via the run script diff --git a/ci/omit-cuslide-benchmarks-and-tests.patch b/ci/omit-cuslide-benchmarks-and-tests.patch new file mode 100644 index 000000000..6394bf2ce --- /dev/null +++ b/ci/omit-cuslide-benchmarks-and-tests.patch @@ -0,0 +1,25 @@ +diff --git a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt +index 37dbca4..18b301a 100644 +--- a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt ++++ b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt +@@ -258,16 +258,16 @@ set_target_properties(${CUCIM_PLUGIN_NAME} PROPERTIES OUTPUT_NAME "${CUCIM_PLUGI + ################################################################################ + # Add tests + #########################################################std####################### +-add_subdirectory(tests) +-add_subdirectory(benchmarks) ++# add_subdirectory(tests) ++# add_subdirectory(benchmarks) + + ################################################################################ + # Install + ################################################################################ + set(INSTALL_TARGETS + ${CUCIM_PLUGIN_NAME} +- cuslide_tests +- cuslide_benchmarks ++ # cuslide_tests ++ # cuslide_benchmarks + ) + + install(TARGETS ${INSTALL_TARGETS} From e381d112c2ec3c3b5abb1455b509baff5d948a13 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 23:14:26 -0400 Subject: [PATCH 13/63] omit building cucim_benchmarks, cucim_tests on arm64 --- ci/omit-cuslide-benchmarks-and-tests.patch | 65 ++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/ci/omit-cuslide-benchmarks-and-tests.patch b/ci/omit-cuslide-benchmarks-and-tests.patch index 6394bf2ce..01c85eb8e 100644 --- a/ci/omit-cuslide-benchmarks-and-tests.patch +++ b/ci/omit-cuslide-benchmarks-and-tests.patch @@ -1,3 +1,68 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 0ab2e17..2a42462 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -138,7 +138,7 @@ set(CUCIM_PACKAGE_NAME cucim) # cucim + ################################################################################ + add_subdirectory(cpp) + add_subdirectory(gds) +-add_subdirectory(benchmarks) ++# add_subdirectory(benchmarks) + add_subdirectory(examples/cpp) + + ################################################################################ +@@ -155,7 +155,7 @@ configure_file(${CMAKE_CURRENT_LIST_DIR}/examples/cpp/CMakeLists.txt.examples.re + set(INSTALL_TARGETS + ${CUCIM_PACKAGE_NAME} + fmt-header-only +- cucim_benchmarks ++ # cucim_benchmarks + ) + + install(TARGETS ${INSTALL_TARGETS} +diff --git a/ci/omit-cuslide-benchmarks-and-tests.patch b/ci/omit-cuslide-benchmarks-and-tests.patch +index 6394bf2..e69de29 100644 +--- a/ci/omit-cuslide-benchmarks-and-tests.patch ++++ b/ci/omit-cuslide-benchmarks-and-tests.patch +@@ -1,25 +0,0 @@ +-diff --git a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt +-index 37dbca4..18b301a 100644 +---- a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt +-+++ b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt +-@@ -258,16 +258,16 @@ set_target_properties(${CUCIM_PLUGIN_NAME} PROPERTIES OUTPUT_NAME "${CUCIM_PLUGI +- ################################################################################ +- # Add tests +- #########################################################std####################### +--add_subdirectory(tests) +--add_subdirectory(benchmarks) +-+# add_subdirectory(tests) +-+# add_subdirectory(benchmarks) +- +- ################################################################################ +- # Install +- ################################################################################ +- set(INSTALL_TARGETS +- ${CUCIM_PLUGIN_NAME} +-- cuslide_tests +-- cuslide_benchmarks +-+ # cuslide_tests +-+ # cuslide_benchmarks +- ) +- +- install(TARGETS ${INSTALL_TARGETS} +diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt +index 67409a6..d2230b3 100644 +--- a/cpp/CMakeLists.txt ++++ b/cpp/CMakeLists.txt +@@ -214,7 +214,7 @@ add_library(${CUCIM_PACKAGE_NAME}::${CUCIM_PACKAGE_NAME}-header-only ALIAS ${CUC + ################################################################################ + # Add tests + ################################################################################ +-add_subdirectory(tests) ++# add_subdirectory(tests) + + ################################################################################# + ## Add bindings diff --git a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt index 37dbca4..18b301a 100644 --- a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt From d932e9842d2380d7469bc05ff96e11b879d7f91f Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 26 Oct 2023 23:50:14 -0400 Subject: [PATCH 14/63] fix patch --- ci/omit-cuslide-benchmarks-and-tests.patch | 38 +++------------------- 1 file changed, 5 insertions(+), 33 deletions(-) diff --git a/ci/omit-cuslide-benchmarks-and-tests.patch b/ci/omit-cuslide-benchmarks-and-tests.patch index 01c85eb8e..f31f50e35 100644 --- a/ci/omit-cuslide-benchmarks-and-tests.patch +++ b/ci/omit-cuslide-benchmarks-and-tests.patch @@ -20,36 +20,6 @@ index 0ab2e17..2a42462 100644 ) install(TARGETS ${INSTALL_TARGETS} -diff --git a/ci/omit-cuslide-benchmarks-and-tests.patch b/ci/omit-cuslide-benchmarks-and-tests.patch -index 6394bf2..e69de29 100644 ---- a/ci/omit-cuslide-benchmarks-and-tests.patch -+++ b/ci/omit-cuslide-benchmarks-and-tests.patch -@@ -1,25 +0,0 @@ --diff --git a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt --index 37dbca4..18b301a 100644 ----- a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt --+++ b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt --@@ -258,16 +258,16 @@ set_target_properties(${CUCIM_PLUGIN_NAME} PROPERTIES OUTPUT_NAME "${CUCIM_PLUGI -- ################################################################################ -- # Add tests -- #########################################################std####################### ---add_subdirectory(tests) ---add_subdirectory(benchmarks) --+# add_subdirectory(tests) --+# add_subdirectory(benchmarks) -- -- ################################################################################ -- # Install -- ################################################################################ -- set(INSTALL_TARGETS -- ${CUCIM_PLUGIN_NAME} --- cuslide_tests --- cuslide_benchmarks --+ # cuslide_tests --+ # cuslide_benchmarks -- ) -- -- install(TARGETS ${INSTALL_TARGETS} diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 67409a6..d2230b3 100644 --- a/cpp/CMakeLists.txt @@ -64,15 +34,17 @@ index 67409a6..d2230b3 100644 ################################################################################# ## Add bindings diff --git a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt -index 37dbca4..18b301a 100644 +index 37dbca4..3cb57a6 100644 --- a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt +++ b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt -@@ -258,16 +258,16 @@ set_target_properties(${CUCIM_PLUGIN_NAME} PROPERTIES OUTPUT_NAME "${CUCIM_PLUGI +@@ -257,17 +257,17 @@ set_target_properties(${CUCIM_PLUGIN_NAME} PROPERTIES OUTPUT_NAME "${CUCIM_PLUGI + ################################################################################ # Add tests - #########################################################std####################### +-#########################################################std####################### -add_subdirectory(tests) -add_subdirectory(benchmarks) ++################################################################################ +# add_subdirectory(tests) +# add_subdirectory(benchmarks) From edec7a03c274cb3e66880e37832be5f1306c3ae9 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 27 Oct 2023 00:03:05 -0400 Subject: [PATCH 15/63] omit copying bin folder --- ci/build_wheel.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 29aa4d860..f2e7ea3be 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -55,11 +55,13 @@ fi # Build the C++ cuslide and cumed plugins ./run build_local cuslide ${CMAKE_BUILD_TYPE} cp -P -r cpp/plugins/cucim.kit.cuslide/install/lib/* ./install/lib/ -cp -P -r cpp/plugins/cucim.kit.cuslide/install/bin/* ./install/bin/ +# omit copying potentially non-existent binaries due to patch above. +# they don't go in the wheel anyways +# cp -P -r cpp/plugins/cucim.kit.cuslide/install/bin/* ./install/bin/ ./run build_local cumed ${CMAKE_BUILD_TYPE} cp -P -r cpp/plugins/cucim.kit.cumed/install/lib/* ./install/lib/ -cp -P -r cpp/plugins/cucim.kit.cumed/install/bin/* ./install/bin/ +# cp -P -r cpp/plugins/cucim.kit.cumed/install/bin/* ./install/bin/ # problems: boost-header-only takes a long time to download From 9f371cfb5e5176f66b8b3592a4bbe7b95a1b0ec3 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 27 Oct 2023 00:43:14 -0400 Subject: [PATCH 16/63] update dependencies.yaml to remove non-pypi files from pyproject tests requirements --- dependencies.yaml | 10 +++++----- python/cucim/pyproject.toml | 5 ----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index fddf3add9..9d9f00ace 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -266,11 +266,11 @@ dependencies: # listed as optional. They are needed in order to run the full test # suite, including the `cucim.clara` package. - click - - jbig - - libwebp-base - - xz - - zlib - - zstd + # - jbig + # - libwebp-base + # - xz + # - zlib + # - zstd # Not sure where these go, if anywhere: # - openslide # - xorg-libxcb diff --git a/python/cucim/pyproject.toml b/python/cucim/pyproject.toml index 2b3143ff7..652ff38ed 100644 --- a/python/cucim/pyproject.toml +++ b/python/cucim/pyproject.toml @@ -63,8 +63,6 @@ test = [ "GPUtil>=1.4.0", "click", "imagecodecs>=2021.6.8", - "jbig", - "libwebp-base", "opencv-python-headless>=4.6", "openslide-python>=1.1.2", "psutil>=5.8.0", @@ -73,9 +71,6 @@ test = [ "pytest-xdist", "pytest>=6.2.4", "tifffile>=2022.7.28", - "xz", - "zlib", - "zstd", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. developer = [ "black", From b70fce1c2bbd101bfb6fc0eb35b19e0ff5107dd2 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 27 Oct 2023 01:19:04 -0400 Subject: [PATCH 17/63] remove imagecodecs and openslide-python test dependencies --- ci/test_wheel.sh | 6 ++++-- dependencies.yaml | 11 +++++++++-- python/cucim/pyproject.toml | 2 -- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index a608a0eb7..a21832fc9 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -13,5 +13,7 @@ python -m pip install $(echo ./dist/cucim*.whl)[test] if [ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]; then python ./ci/wheel_smoke_test.py else - python -m pytest ./python/cucim -fi + # python -m pytest ./python/cucim + # omit clara subset until dependency installs are fixed + python -m pytest ./python/cucim/src/cucim/core/ + python -m pytest ./python/cucim/src/cucim/skimage/ diff --git a/dependencies.yaml b/dependencies.yaml index 9d9f00ace..f34578ca2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -245,8 +245,6 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - GPUtil>=1.4.0 - - imagecodecs>=2021.6.8 - - openslide-python>=1.1.2 - psutil>=5.8.0 - pytest-cov>=2.12.1 - pytest-lazy-fixture>=0.6.3 @@ -255,9 +253,18 @@ dependencies: - tifffile>=2022.7.28 - output_types: [conda] packages: + # TODO: fix dependencies and move imagecodecs & openslide-python back up to the common + # section above. (doesn't seem that imagecodecs has an arm64 wheel) + - imagecodecs>=2021.6.8 + - openslide-python>=1.1.2 - pip - pip: - opencv-python-headless>=4.6 + - output_types: [requirements] + packages: + # TODO: fix dependencies and move this back up to the common section above + - imagecodecs>=2021.6.8 + - openslide-python>=1.1.2 - output_types: [requirements, pyproject] packages: - opencv-python-headless>=4.6 diff --git a/python/cucim/pyproject.toml b/python/cucim/pyproject.toml index 652ff38ed..5df4f4815 100644 --- a/python/cucim/pyproject.toml +++ b/python/cucim/pyproject.toml @@ -62,9 +62,7 @@ Tracker = "https://github.com/rapidsai/cucim/issues" test = [ "GPUtil>=1.4.0", "click", - "imagecodecs>=2021.6.8", "opencv-python-headless>=4.6", - "openslide-python>=1.1.2", "psutil>=5.8.0", "pytest-cov>=2.12.1", "pytest-lazy-fixture>=0.6.3", From 4a1d09ea4d35f621d249fa585ba82a20e65f65bb Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 27 Oct 2023 08:21:47 -0400 Subject: [PATCH 18/63] add missing fi in if/else statement --- ci/test_wheel.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index a21832fc9..e9ae38261 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -17,3 +17,4 @@ else # omit clara subset until dependency installs are fixed python -m pytest ./python/cucim/src/cucim/core/ python -m pytest ./python/cucim/src/cucim/skimage/ +fi From 4c4f4a0a7d851766717fbdd9a8cd8088193ac90c Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 27 Oct 2023 09:01:52 -0400 Subject: [PATCH 19/63] modify CuPy package name as needed based on the CUDA version --- ci/build_wheel.sh | 12 +++--------- ci/release/apply_wheel_modifications.sh | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 9 deletions(-) create mode 100755 ci/release/apply_wheel_modifications.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index f2e7ea3be..61b12e0fe 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -17,15 +17,9 @@ version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -# Patch project metadata files to include the CUDA version suffix and version override. -pyproject_file="${package_dir}/pyproject.toml" - -sed -i "s/^version = .*/version = \"${version_override}\"/g" ${pyproject_file} -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}" +echo "The package name and/or version was modified in the package source. The git diff is:" +git diff pip install --upgrade pip diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh new file mode 100755 index 000000000..401c6e605 --- /dev/null +++ b/ci/release/apply_wheel_modifications.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Usage: bash apply_wheel_modifications.sh + +VERSION=${1} +CUDA_SUFFIX=${2} + +package_name="cucim" +package_dir="python/cucim" + +# Patch project metadata files to include the CUDA version suffix and version override. +pyproject_file="${package_dir}/pyproject.toml" + +# pyproject.toml versions +sed -i "s/^version = .*/version = \"${VERSION}\"/g" ${pyproject_file} + +# update package name to have the cuda suffix +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${CUDA_SUFFIX}\"/g" ${pyproject_file} + +if [[ $CUDA_SUFFIX == "-cu12" ]]; then + # change pyproject.toml to use CUDA 12.x version of cupy + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} +fi From 977f67246202f563aeebf15827082b46deb88081 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 27 Oct 2023 09:38:26 -0400 Subject: [PATCH 20/63] allow test suite to run even if imagecodecs was not installed --- .../tests/performance/clara/test_read_region_memory_usage.py | 3 +++ python/cucim/tests/unit/clara/converter/test_converter.py | 5 +++++ python/cucim/tests/unit/clara/test_image_cache.py | 5 +++++ python/cucim/tests/unit/clara/test_load_image_metadata.py | 5 +++++ python/cucim/tests/unit/clara/test_tiff_read_region.py | 3 +++ 5 files changed, 21 insertions(+) diff --git a/python/cucim/tests/performance/clara/test_read_region_memory_usage.py b/python/cucim/tests/performance/clara/test_read_region_memory_usage.py index aba340eea..b53bdf4b8 100644 --- a/python/cucim/tests/performance/clara/test_read_region_memory_usage.py +++ b/python/cucim/tests/performance/clara/test_read_region_memory_usage.py @@ -17,6 +17,9 @@ from ...util.io import open_image_cucim +# skip if imagecodecs package not available (needed by ImageGenerator utility) +pytest.importorskip("imagecodecs") + def test_read_region_cuda_memleak(testimg_tiff_stripe_4096x4096_256_jpeg): import GPUtil diff --git a/python/cucim/tests/unit/clara/converter/test_converter.py b/python/cucim/tests/unit/clara/converter/test_converter.py index 6e54640ea..1ded75824 100644 --- a/python/cucim/tests/unit/clara/converter/test_converter.py +++ b/python/cucim/tests/unit/clara/converter/test_converter.py @@ -16,6 +16,11 @@ import os from pathlib import Path +import pytest + +# skip if imagecodecs package not available (needed by ImageGenerator utility) +pytest.importorskip("imagecodecs") + def test_image_converter_stripe_4096x4096_256_jpeg( tmp_path, testimg_tiff_stripe_4096x4096_256_jpeg diff --git a/python/cucim/tests/unit/clara/test_image_cache.py b/python/cucim/tests/unit/clara/test_image_cache.py index f6f7d199f..1b8987765 100644 --- a/python/cucim/tests/unit/clara/test_image_cache.py +++ b/python/cucim/tests/unit/clara/test_image_cache.py @@ -13,6 +13,11 @@ # limitations under the License. # +import pytest + +# skip if imagecodecs package not available (needed by ImageGenerator utility) +pytest.importorskip("imagecodecs") + def test_get_nocache(): from cucim import CuImage diff --git a/python/cucim/tests/unit/clara/test_load_image_metadata.py b/python/cucim/tests/unit/clara/test_load_image_metadata.py index 971e4e171..cc434c4bd 100644 --- a/python/cucim/tests/unit/clara/test_load_image_metadata.py +++ b/python/cucim/tests/unit/clara/test_load_image_metadata.py @@ -15,8 +15,13 @@ import math +import pytest + from ...util.io import open_image_cucim +# skip if imagecodecs package not available (needed by ImageGenerator utility) +pytest.importorskip("imagecodecs") + def test_load_image_metadata(testimg_tiff_stripe_32x24_16): import numpy as np diff --git a/python/cucim/tests/unit/clara/test_tiff_read_region.py b/python/cucim/tests/unit/clara/test_tiff_read_region.py index 9fafe431c..86cd4c5f2 100644 --- a/python/cucim/tests/unit/clara/test_tiff_read_region.py +++ b/python/cucim/tests/unit/clara/test_tiff_read_region.py @@ -18,6 +18,9 @@ from ...util.io import open_image_cucim +# skip if imagecodecs package not available (needed by ImageGenerator utility) +pytest.importorskip("imagecodecs") + def test_tiff_stripe_inner(testimg_tiff_stripe_32x24_16): cucim_img = open_image_cucim(testimg_tiff_stripe_32x24_16) From 5f87e2ad4331daa8c7b0684240242356617d8abc Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 27 Oct 2023 10:07:57 -0400 Subject: [PATCH 21/63] fix path for libopenslide on Ubuntu+aarch64 --- ci/build_wheel.sh | 4 ++-- ci/test_wheel.sh | 7 +++---- cpp/cmake/deps/openslide.cmake | 4 +++- cpp/plugins/cucim.kit.cuslide/cmake/deps/openslide.cmake | 4 +++- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 61b12e0fe..18ad725ab 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -38,9 +38,9 @@ else echo "apt package manager was found" apt update apt install yasm libopenslide-dev -y - dpkg -L libopenslide-dev + # dpkg -L libopenslide-dev - git apply ci/omit-cuslide-benchmarks-and-tests.patch + # git apply ci/omit-cuslide-benchmarks-and-tests.patch fi # First build the C++ lib using CMake via the run script diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index e9ae38261..ee5b3264d 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -13,8 +13,7 @@ python -m pip install $(echo ./dist/cucim*.whl)[test] if [ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]; then python ./ci/wheel_smoke_test.py else - # python -m pytest ./python/cucim - # omit clara subset until dependency installs are fixed - python -m pytest ./python/cucim/src/cucim/core/ - python -m pytest ./python/cucim/src/cucim/skimage/ + # Note: imagecodecs is currently not installed on aarch64, so a handful of + # test cases will be skipped in that case + python -m pytest ./python/cucim fi diff --git a/cpp/cmake/deps/openslide.cmake b/cpp/cmake/deps/openslide.cmake index 651acd5ff..8e44e33a5 100644 --- a/cpp/cmake/deps/openslide.cmake +++ b/cpp/cmake/deps/openslide.cmake @@ -22,7 +22,9 @@ if (NOT TARGET deps::openslide) set(OPENSLIDE_LIB_PATH "$ENV{CONDA_PREFIX}/lib/libopenslide.so") elseif (EXISTS /usr/lib/x86_64-linux-gnu/libopenslide.so) set(OPENSLIDE_LIB_PATH /usr/lib/x86_64-linux-gnu/libopenslide.so) - else () # CentOS 6 + elseif (EXISTS /usr/lib/aarch64-linux-gnu/libopenslide.so) + set(OPENSLIDE_LIB_PATH /usr/lib/aarch64-linux-gnu/libopenslide.so) + else () # CentOS (x86_64) set(OPENSLIDE_LIB_PATH /usr/lib64/libopenslide.so) endif () diff --git a/cpp/plugins/cucim.kit.cuslide/cmake/deps/openslide.cmake b/cpp/plugins/cucim.kit.cuslide/cmake/deps/openslide.cmake index 651acd5ff..8e44e33a5 100644 --- a/cpp/plugins/cucim.kit.cuslide/cmake/deps/openslide.cmake +++ b/cpp/plugins/cucim.kit.cuslide/cmake/deps/openslide.cmake @@ -22,7 +22,9 @@ if (NOT TARGET deps::openslide) set(OPENSLIDE_LIB_PATH "$ENV{CONDA_PREFIX}/lib/libopenslide.so") elseif (EXISTS /usr/lib/x86_64-linux-gnu/libopenslide.so) set(OPENSLIDE_LIB_PATH /usr/lib/x86_64-linux-gnu/libopenslide.so) - else () # CentOS 6 + elseif (EXISTS /usr/lib/aarch64-linux-gnu/libopenslide.so) + set(OPENSLIDE_LIB_PATH /usr/lib/aarch64-linux-gnu/libopenslide.so) + else () # CentOS (x86_64) set(OPENSLIDE_LIB_PATH /usr/lib64/libopenslide.so) endif () From 6153d1bf1e57ebf8f6359ae3dbb3b351fd35cd98 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 27 Oct 2023 15:27:28 -0400 Subject: [PATCH 22/63] remove unused patch --- ci/build_wheel.sh | 13 ++--- ci/omit-cuslide-benchmarks-and-tests.patch | 62 ---------------------- dependencies.yaml | 10 +--- 3 files changed, 4 insertions(+), 81 deletions(-) delete mode 100644 ci/omit-cuslide-benchmarks-and-tests.patch diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 18ad725ab..3b4b034f1 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -38,9 +38,6 @@ else echo "apt package manager was found" apt update apt install yasm libopenslide-dev -y - # dpkg -L libopenslide-dev - - # git apply ci/omit-cuslide-benchmarks-and-tests.patch fi # First build the C++ lib using CMake via the run script @@ -49,22 +46,18 @@ fi # Build the C++ cuslide and cumed plugins ./run build_local cuslide ${CMAKE_BUILD_TYPE} cp -P -r cpp/plugins/cucim.kit.cuslide/install/lib/* ./install/lib/ -# omit copying potentially non-existent binaries due to patch above. -# they don't go in the wheel anyways -# cp -P -r cpp/plugins/cucim.kit.cuslide/install/bin/* ./install/bin/ +# omit copying binaries as they don't go in the wheel ./run build_local cumed ${CMAKE_BUILD_TYPE} cp -P -r cpp/plugins/cucim.kit.cumed/install/lib/* ./install/lib/ -# cp -P -r cpp/plugins/cucim.kit.cumed/install/bin/* ./install/bin/ - -# problems: boost-header-only takes a long time to download +# omit copying binaries as they don't go in the wheel # Compile the Python bindings ./run build_local cucim ${CMAKE_BUILD_TYPE} # Copy the resulting cucim pybind11 shared library into the Python package src folder cp -P python/install/lib/* python/cucim/src/cucim/clara/ -# also need these files in the clara wheel +# also need these plugin / libcucim shared libraries in the clara wheel cp -P install/lib/*.so python/cucim/src/cucim/clara/ cd "${package_dir}" diff --git a/ci/omit-cuslide-benchmarks-and-tests.patch b/ci/omit-cuslide-benchmarks-and-tests.patch deleted file mode 100644 index f31f50e35..000000000 --- a/ci/omit-cuslide-benchmarks-and-tests.patch +++ /dev/null @@ -1,62 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index 0ab2e17..2a42462 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -138,7 +138,7 @@ set(CUCIM_PACKAGE_NAME cucim) # cucim - ################################################################################ - add_subdirectory(cpp) - add_subdirectory(gds) --add_subdirectory(benchmarks) -+# add_subdirectory(benchmarks) - add_subdirectory(examples/cpp) - - ################################################################################ -@@ -155,7 +155,7 @@ configure_file(${CMAKE_CURRENT_LIST_DIR}/examples/cpp/CMakeLists.txt.examples.re - set(INSTALL_TARGETS - ${CUCIM_PACKAGE_NAME} - fmt-header-only -- cucim_benchmarks -+ # cucim_benchmarks - ) - - install(TARGETS ${INSTALL_TARGETS} -diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt -index 67409a6..d2230b3 100644 ---- a/cpp/CMakeLists.txt -+++ b/cpp/CMakeLists.txt -@@ -214,7 +214,7 @@ add_library(${CUCIM_PACKAGE_NAME}::${CUCIM_PACKAGE_NAME}-header-only ALIAS ${CUC - ################################################################################ - # Add tests - ################################################################################ --add_subdirectory(tests) -+# add_subdirectory(tests) - - ################################################################################# - ## Add bindings -diff --git a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt -index 37dbca4..3cb57a6 100644 ---- a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt -+++ b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt -@@ -257,17 +257,17 @@ set_target_properties(${CUCIM_PLUGIN_NAME} PROPERTIES OUTPUT_NAME "${CUCIM_PLUGI - - ################################################################################ - # Add tests --#########################################################std####################### --add_subdirectory(tests) --add_subdirectory(benchmarks) -+################################################################################ -+# add_subdirectory(tests) -+# add_subdirectory(benchmarks) - - ################################################################################ - # Install - ################################################################################ - set(INSTALL_TARGETS - ${CUCIM_PLUGIN_NAME} -- cuslide_tests -- cuslide_benchmarks -+ # cuslide_tests -+ # cuslide_benchmarks - ) - - install(TARGETS ${INSTALL_TARGETS} diff --git a/dependencies.yaml b/dependencies.yaml index f34578ca2..edac5d4a6 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -253,7 +253,7 @@ dependencies: - tifffile>=2022.7.28 - output_types: [conda] packages: - # TODO: fix dependencies and move imagecodecs & openslide-python back up to the common + # TODO: move imagecodecs & openslide-python back up to the common # section above. (doesn't seem that imagecodecs has an arm64 wheel) - imagecodecs>=2021.6.8 - openslide-python>=1.1.2 @@ -273,11 +273,3 @@ dependencies: # listed as optional. They are needed in order to run the full test # suite, including the `cucim.clara` package. - click - # - jbig - # - libwebp-base - # - xz - # - zlib - # - zstd - # Not sure where these go, if anywhere: - # - openslide - # - xorg-libxcb From cf2ca0e1a1913e9e6950f313c18fe4fb3d880376 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 27 Oct 2023 18:14:11 -0400 Subject: [PATCH 23/63] install imagecodecs and openslide-python on x86_64 to enable more test cases --- dependencies.yaml | 12 +++++------- python/cucim/pyproject.toml | 2 ++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index edac5d4a6..3e098a359 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -253,8 +253,6 @@ dependencies: - tifffile>=2022.7.28 - output_types: [conda] packages: - # TODO: move imagecodecs & openslide-python back up to the common - # section above. (doesn't seem that imagecodecs has an arm64 wheel) - imagecodecs>=2021.6.8 - openslide-python>=1.1.2 - pip @@ -262,14 +260,14 @@ dependencies: - opencv-python-headless>=4.6 - output_types: [requirements] packages: - # TODO: fix dependencies and move this back up to the common section above - imagecodecs>=2021.6.8 - openslide-python>=1.1.2 + - output_types: [pyproject] + packages: + # skip packages on arm64 that don't provide a wheel + - imagecodecs>=2021.6.8; platform_machine=='x86_64' + - openslide-python>=1.1.2; platform_machine=='x86_64' - output_types: [requirements, pyproject] packages: - opencv-python-headless>=4.6 - # All dependencies below this point are specific to `cucim.clara` and - # are not needed for either `cucim.core` or `cucim.skimage`, so are - # listed as optional. They are needed in order to run the full test - # suite, including the `cucim.clara` package. - click diff --git a/python/cucim/pyproject.toml b/python/cucim/pyproject.toml index 5df4f4815..83a64dca7 100644 --- a/python/cucim/pyproject.toml +++ b/python/cucim/pyproject.toml @@ -62,7 +62,9 @@ Tracker = "https://github.com/rapidsai/cucim/issues" test = [ "GPUtil>=1.4.0", "click", + "imagecodecs>=2021.6.8; platform_machine=='x86_64'", "opencv-python-headless>=4.6", + "openslide-python>=1.1.2; platform_machine=='x86_64'", "psutil>=5.8.0", "pytest-cov>=2.12.1", "pytest-lazy-fixture>=0.6.3", From e8721858397952ee82a0d0290c81f73275f6e3c1 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Sat, 28 Oct 2023 09:41:25 -0400 Subject: [PATCH 24/63] try installing libopenslide-dev during wheel tests as well --- ci/test_wheel.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index ee5b3264d..4d6e8b399 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -6,6 +6,20 @@ set -eou pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cucim_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist +# some test cases require libopenslide.so for writing the test TIFF images. + +# Building the libjpeg-turbo dependency requires YASM +# Also need to install openslide dev libraries on the system +if ! command -v apt &> /dev/null +then + echo "apt package manager not found, attempting to use yum" + yum install openslide-devel -y +else + echo "apt package manager was found" + apt update + apt install libopenslide-dev -y +fi + # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/cucim*.whl)[test] From 9c4d5ce2545689a67babf5faf0ffd64cc408c969 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Sat, 28 Oct 2023 15:36:47 -0400 Subject: [PATCH 25/63] skip single test case requiring libopenslide --- ci/test_wheel.sh | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 4d6e8b399..552b11ae8 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -6,20 +6,6 @@ set -eou pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="cucim_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# some test cases require libopenslide.so for writing the test TIFF images. - -# Building the libjpeg-turbo dependency requires YASM -# Also need to install openslide dev libraries on the system -if ! command -v apt &> /dev/null -then - echo "apt package manager not found, attempting to use yum" - yum install openslide-devel -y -else - echo "apt package manager was found" - apt update - apt install libopenslide-dev -y -fi - # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/cucim*.whl)[test] @@ -27,7 +13,7 @@ python -m pip install $(echo ./dist/cucim*.whl)[test] if [ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]; then python ./ci/wheel_smoke_test.py else - # Note: imagecodecs is currently not installed on aarch64, so a handful of - # test cases will be skipped in that case - python -m pytest ./python/cucim + # skip `test_converter`` (which requires openslide-python and libopenslide) + # (a segfault was observed when using the system libopenslide-dev) + python -m pytest ./python/cucim -k "not test_converter" fi From 46254bcaefd6ce527b35dbc8dbcbd09e598a614c Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Sun, 29 Oct 2023 09:11:21 -0400 Subject: [PATCH 26/63] add additional requirements used by a small number of test cases --- conda/environments/all_cuda-118_arch-x86_64.yaml | 2 ++ conda/environments/all_cuda-120_arch-x86_64.yaml | 2 ++ dependencies.yaml | 2 ++ python/cucim/pyproject.toml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 19d15f9a6..7c5083410 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -24,6 +24,7 @@ dependencies: - libnvjpeg-dev=11.6.0.55 - libnvjpeg=11.6.0.55 - libwebp-base +- matplotlib - nbsphinx - ninja - numpy>=1.21.3 @@ -31,6 +32,7 @@ dependencies: - nvcc_linux-64=11.8 - openslide-python>=1.1.2 - pip +- pooch>=1.6.0 - pre-commit - psutil>=5.8.0 - pydata-sphinx-theme diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 982a2c627..13f0e01a1 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -24,12 +24,14 @@ dependencies: - libnvjpeg-dev - libnvjpeg-static - libwebp-base +- matplotlib - nbsphinx - ninja - numpy>=1.21.3 - numpydoc - openslide-python>=1.1.2 - pip +- pooch>=1.6.0 - pre-commit - psutil>=5.8.0 - pydata-sphinx-theme diff --git a/dependencies.yaml b/dependencies.yaml index 3e098a359..6f9f7f93b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -251,6 +251,8 @@ dependencies: - pytest-xdist - pytest>=6.2.4 - tifffile>=2022.7.28 + - pooch>=1.6.0 # needed to download scikit-image sample data + - matplotlib - output_types: [conda] packages: - imagecodecs>=2021.6.8 diff --git a/python/cucim/pyproject.toml b/python/cucim/pyproject.toml index 83a64dca7..8657d6da6 100644 --- a/python/cucim/pyproject.toml +++ b/python/cucim/pyproject.toml @@ -63,8 +63,10 @@ test = [ "GPUtil>=1.4.0", "click", "imagecodecs>=2021.6.8; platform_machine=='x86_64'", + "matplotlib", "opencv-python-headless>=4.6", "openslide-python>=1.1.2; platform_machine=='x86_64'", + "pooch>=1.6.0", "psutil>=5.8.0", "pytest-cov>=2.12.1", "pytest-lazy-fixture>=0.6.3", From 076964a67e88194cabeb68eb34ec1e299a0b4164 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Sun, 29 Oct 2023 09:14:50 -0400 Subject: [PATCH 27/63] check if clara test cases are causing the segfault observed on CUDA 11.8 x86_64 --- .github/workflows/pr.yaml | 4 ++++ ci/test_wheel.sh | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d098ff326..1a164e759 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -66,3 +66,7 @@ jobs: with: build_type: pull-request script: ci/test_wheel.sh + uses: actions/cache@v3 + with: + path: ~/.cache/kernel_cache + key: cucim-kernel-cache-0|${{ hashFiles('.pre-commit-config.yaml') }} diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 552b11ae8..a44e2019d 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -14,6 +14,12 @@ if [ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]; then python ./ci/wheel_smoke_test.py else # skip `test_converter`` (which requires openslide-python and libopenslide) - # (a segfault was observed when using the system libopenslide-dev) - python -m pytest ./python/cucim -k "not test_converter" + # (A segfault was observed when using the system libopenslide-dev. + # No segfault was seen when testing locally using conda-forge's openslide-python) + if [[ ${RAPIDS_PY_CUDA_SUFFIX} == "cu12" ]]; then + python -m pytest ./python/cucim -k "not test_converter" + else + # temporarily disable all clara tests to check if segfault is avoided + python -m pytest ./python/cucim -k "not clara" + fi fi From f22f45c742846098ae76b300d923e0908e85824d Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Mon, 30 Oct 2023 13:42:23 -0400 Subject: [PATCH 28/63] revert unintentional workflow file change --- .github/workflows/pr.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1a164e759..d098ff326 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -66,7 +66,3 @@ jobs: with: build_type: pull-request script: ci/test_wheel.sh - uses: actions/cache@v3 - with: - path: ~/.cache/kernel_cache - key: cucim-kernel-cache-0|${{ hashFiles('.pre-commit-config.yaml') }} From 1d088c794bd4558481ac1db5eb2fc30a400680a8 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Mon, 30 Oct 2023 15:10:01 -0400 Subject: [PATCH 29/63] make sure which is installed on rockylinux8 --- ci/build_wheel.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 3b4b034f1..472519f47 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -26,20 +26,22 @@ pip install --upgrade pip #CMake version in the container is too old, install a new version in the python venv pip install "cmake>=3.26.4" ninja -echo `which cmake` - # Building the libjpeg-turbo dependency requires YASM # Also need to install openslide dev libraries on the system if ! command -v apt &> /dev/null then echo "apt package manager not found, attempting to use yum" - yum install yasm openslide-devel -y + yum install yasm openslide-devel which -y else echo "apt package manager was found" apt update apt install yasm libopenslide-dev -y fi +echo `which cmake` +echo `which python` +echo `python --version` + # First build the C++ lib using CMake via the run script ./run build_local libcucim ${CMAKE_BUILD_TYPE} From 7ce3653e76ad1a7b84f0f31c2b1d0a95d9613994 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Mon, 30 Oct 2023 15:10:39 -0400 Subject: [PATCH 30/63] bumpy pybind11 version --- cpp/cmake/deps/pybind11.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/deps/pybind11.cmake b/cpp/cmake/deps/pybind11.cmake index 1fbba6c9d..2bab5c8b4 100644 --- a/cpp/cmake/deps/pybind11.cmake +++ b/cpp/cmake/deps/pybind11.cmake @@ -17,7 +17,7 @@ if (NOT TARGET deps::pybind11) FetchContent_Declare( deps-pybind11 GIT_REPOSITORY https://github.com/pybind/pybind11.git - GIT_TAG v2.10.3 + GIT_TAG v2.11.1 GIT_SHALLOW TRUE ) FetchContent_GetProperties(deps-pybind11) From 3b058c7720df558e3a1df933706ed275b2a83b5b Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Mon, 30 Oct 2023 16:40:56 -0400 Subject: [PATCH 31/63] which was added to the container, so can remove it from the build script minor cleanup of ci/build_wheel.sh --- ci/build_wheel.sh | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 472519f47..80814181e 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -31,36 +31,29 @@ pip install "cmake>=3.26.4" ninja if ! command -v apt &> /dev/null then echo "apt package manager not found, attempting to use yum" - yum install yasm openslide-devel which -y + yum install yasm openslide-devel -y else echo "apt package manager was found" apt update apt install yasm libopenslide-dev -y fi -echo `which cmake` -echo `which python` -echo `python --version` - # First build the C++ lib using CMake via the run script ./run build_local libcucim ${CMAKE_BUILD_TYPE} +# Current `pip wheel` build assumes all shared libraries exist in this path +PYTHON_CLARA_SO_PREFIX_PATH="./python/cucim/src/cucim/clara/" + # Build the C++ cuslide and cumed plugins +# (omit copying binaries as they don't go in the wheel) ./run build_local cuslide ${CMAKE_BUILD_TYPE} -cp -P -r cpp/plugins/cucim.kit.cuslide/install/lib/* ./install/lib/ -# omit copying binaries as they don't go in the wheel - +cp -P -r cpp/plugins/cucim.kit.cuslide/install/lib/* ${PYTHON_CLARA_SO_PREFIX_PATH} ./run build_local cumed ${CMAKE_BUILD_TYPE} -cp -P -r cpp/plugins/cucim.kit.cumed/install/lib/* ./install/lib/ -# omit copying binaries as they don't go in the wheel +cp -P -r cpp/plugins/cucim.kit.cumed/install/lib/* ${PYTHON_CLARA_SO_PREFIX_PATH} # Compile the Python bindings ./run build_local cucim ${CMAKE_BUILD_TYPE} - -# Copy the resulting cucim pybind11 shared library into the Python package src folder -cp -P python/install/lib/* python/cucim/src/cucim/clara/ -# also need these plugin / libcucim shared libraries in the clara wheel -cp -P install/lib/*.so python/cucim/src/cucim/clara/ +cp -P install/lib/*.so ${PYTHON_CLARA_SO_PREFIX_PATH} cd "${package_dir}" From 27912280e8492c3ffd84165f4830d826bf755e5c Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 09:57:05 -0400 Subject: [PATCH 32/63] move existing apply_wheel_modifications.sh logic into build_wheel.sh --- ci/build_wheel.sh | 16 +++++++++++++++- ci/release/apply_wheel_modifications.sh | 24 ------------------------ 2 files changed, 15 insertions(+), 25 deletions(-) delete mode 100755 ci/release/apply_wheel_modifications.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 80814181e..c2f237441 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -17,7 +17,21 @@ version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}" +# Patch project metadata files to include the CUDA version suffix and version override. +pyproject_file="${package_dir}/pyproject.toml" + +# pyproject.toml versions +sed -i "s/^version = .*/version = \"${VERSION}\"/g" ${pyproject_file} + +CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" +# update package name to have the cuda suffix +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${CUDA_SUFFIX}\"/g" ${pyproject_file} + +if [[ $CUDA_SUFFIX == "-cu12" ]]; then + # change pyproject.toml to use CUDA 12.x version of cupy + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} +fi + echo "The package name and/or version was modified in the package source. The git diff is:" git diff diff --git a/ci/release/apply_wheel_modifications.sh b/ci/release/apply_wheel_modifications.sh deleted file mode 100755 index 401c6e605..000000000 --- a/ci/release/apply_wheel_modifications.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. -# -# Usage: bash apply_wheel_modifications.sh - -VERSION=${1} -CUDA_SUFFIX=${2} - -package_name="cucim" -package_dir="python/cucim" - -# Patch project metadata files to include the CUDA version suffix and version override. -pyproject_file="${package_dir}/pyproject.toml" - -# pyproject.toml versions -sed -i "s/^version = .*/version = \"${VERSION}\"/g" ${pyproject_file} - -# update package name to have the cuda suffix -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${CUDA_SUFFIX}\"/g" ${pyproject_file} - -if [[ $CUDA_SUFFIX == "-cu12" ]]; then - # change pyproject.toml to use CUDA 12.x version of cupy - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} -fi From dff57f8bcc72c95e6b80c25759c5df79063d642d Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 10:39:52 -0400 Subject: [PATCH 33/63] refactor VERSION management following example in rapidsai/rmm#1347 --- ci/build_cpp.sh | 4 +++- ci/build_python.sh | 11 ++++++++++- ci/build_wheel.sh | 17 ++++++++--------- ci/release/update-version.sh | 11 +++++------ ci/test_wheel.sh | 2 +- conda/recipes/cucim/meta.yaml | 4 ++-- conda/recipes/libcucim/meta.yaml | 4 ++-- python/cucim/VERSION | 1 - python/cucim/pyproject.toml | 9 +++++---- python/cucim/src/cucim/VERSION | 1 + python/cucim/src/cucim/__init__.py | 13 +++++-------- python/cucim/src/cucim/__init__.pyi | 12 +++++++++--- python/cucim/src/cucim/_version.py | 20 ++++++++++++++++++++ python/cucim/src/cucim/clara/__init__.py | 11 +---------- 14 files changed, 72 insertions(+), 48 deletions(-) delete mode 100644 python/cucim/VERSION create mode 120000 python/cucim/src/cucim/VERSION create mode 100644 python/cucim/src/cucim/_version.py diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 1cc80d198..b0ef4a446 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -9,8 +9,10 @@ export CMAKE_GENERATOR=Ninja rapids-print-env +version=$(rapids-generate-version) + rapids-logger "Begin cpp build" -rapids-conda-retry mambabuild conda/recipes/libcucim +RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libcucim rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index ae607aeeb..1597d05fb 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -9,13 +9,22 @@ export CMAKE_GENERATOR=Ninja rapids-print-env +package_name="cucim" +package_dir="python/cucim/src" + +version=$(rapids-generate-version) +commit=$(git rev-parse HEAD) + +echo "${version}" > VERSION +sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" + rapids-logger "Begin py build" CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp) # TODO: Remove `--no-test` flag once importing on a CPU # node works correctly -rapids-conda-retry mambabuild \ +RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ conda/recipes/cucim diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index c2f237441..d88384bd1 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,29 +5,28 @@ set -euo pipefail package_name="cucim" package_dir="python/cucim" +package_src_dir="${package_dir}/src" CMAKE_BUILD_TYPE="release" source rapids-configure-sccache source rapids-date-string -# Use gha-tools rapids-pip-wheel-version to generate wheel version then -# update the necessary files -version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})" +version=$(rapids-generate-version) +commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Patch project metadata files to include the CUDA version suffix and version override. pyproject_file="${package_dir}/pyproject.toml" -# pyproject.toml versions -sed -i "s/^version = .*/version = \"${VERSION}\"/g" ${pyproject_file} - -CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" # update package name to have the cuda suffix -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${CUDA_SUFFIX}\"/g" ${pyproject_file} +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +echo "${version}" > VERSION +sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_src_dir}/_version.py" -if [[ $CUDA_SUFFIX == "-cu12" ]]; then +if [[ ${PACKAGE_CUDA_SUFFIX} == "-cu12" ]]; then # change pyproject.toml to use CUDA 12.x version of cupy sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} fi diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 37cee5a67..b41c38643 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -34,12 +34,11 @@ function sed_runner() { sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/source/conf.py sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf.py -sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cucim/pyproject.toml -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" VERSION -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" python/cucim/VERSION -sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cucim/src/cucim/__init__.pyi -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cuslide/VERSION -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cumed/VERSION +# Centralized version file update +echo "${NEXT_FULL_TAG}" > VERSION +echo "${NEXT_FULL_TAG}" > cpp/plugins/cucim.kit.cuslide/VERSION +echo "${NEXT_FULL_TAG}" > cpp/plugins/cucim.kit.cumed/VERSION + sed_runner "s#\[Version ${CURRENT_LONG_TAG}\](release_notes/v${CURRENT_LONG_TAG}.md)#\[Version ${NEXT_FULL_TAG}\](release_notes/v${NEXT_FULL_TAG}.md)#g" python/cucim/docs/index.md sed_runner "s/v${CURRENT_LONG_TAG}/v${NEXT_FULL_TAG}/g" python/cucim/docs/getting_started/index.md sed_runner "s#cucim.kit.cuslide@${CURRENT_LONG_TAG}.so#cucim.kit.cuslide@${NEXT_FULL_TAG}.so#g" python/cucim/docs/getting_started/index.md diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index a44e2019d..2c71c4751 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -10,7 +10,7 @@ RAPIDS_PY_WHEEL_NAME="cucim_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-fro python -m pip install $(echo ./dist/cucim*.whl)[test] # Run smoke tests for aarch64 pull requests -if [ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]; then +if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then python ./ci/wheel_smoke_test.py else # skip `test_converter`` (which requires openslide-python and libopenslide) diff --git a/conda/recipes/cucim/meta.yaml b/conda/recipes/cucim/meta.yaml index 609ca09c5..dbb34a906 100644 --- a/conda/recipes/cucim/meta.yaml +++ b/conda/recipes/cucim/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2021-2023, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} {% set py_version = environ['CONDA_PY'] %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} @@ -12,7 +12,7 @@ package: version: {{ version }} source: - git_url: ../../.. + path: ../../.. build: number: {{ GIT_DESCRIBE_NUMBER }} diff --git a/conda/recipes/libcucim/meta.yaml b/conda/recipes/libcucim/meta.yaml index 4140bbf1d..666a56d90 100644 --- a/conda/recipes/libcucim/meta.yaml +++ b/conda/recipes/libcucim/meta.yaml @@ -1,6 +1,6 @@ # Copyright (c) 2021-2023, NVIDIA CORPORATION. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %} +{% set version = environ['RAPIDS_PACKAGE_VERSION'].lstrip('v') %} {% set cuda_version = '.'.join(environ['RAPIDS_CUDA_VERSION'].split('.')[:2]) %} {% set cuda_major = cuda_version.split('.')[0] %} {% set date_string = environ['RAPIDS_DATE_STRING'] %} @@ -11,7 +11,7 @@ package: version: {{ version }} source: - git_url: ../../.. + path: ../../.. build: number: {{ GIT_DESCRIBE_NUMBER }} diff --git a/python/cucim/VERSION b/python/cucim/VERSION deleted file mode 100644 index a193fff41..000000000 --- a/python/cucim/VERSION +++ /dev/null @@ -1 +0,0 @@ -23.12.00 diff --git a/python/cucim/pyproject.toml b/python/cucim/pyproject.toml index 8657d6da6..2a6f20e10 100644 --- a/python/cucim/pyproject.toml +++ b/python/cucim/pyproject.toml @@ -10,7 +10,7 @@ requires = [ [project] name = "cucim" -version = "23.12.00" +dynamic = ["version"] description = "cuCIM - an extensible toolkit designed to provide GPU accelerated I/O, computer vision & image processing primitives for N-Dimensional images with a focus on biomedical imaging." # TODO: tried also adding CHANGELOG.md as in setup.py's long_description, but ruff complained about it readme = { file = "README.md", content-type = "text/markdown" } @@ -94,15 +94,16 @@ cucim = "cucim.clara.cli:main" [tool.setuptools] license-files = ["LICENSE"] -# By default, include-package-data is true in pyproject.toml, so you do -# NOT have to specify this line. include-package-data = true +[tool.setuptools.dynamic] +version = {file = "src/cucim/VERSION"} + [tool.setuptools.packages.find] where = ["src"] [tool.setuptools.package-data] -mypkg = ["*.pyi", "*.h", "*.cu"] +mypkg = ["*.pyi", "*.h", "*.cu", "VERSION"] [tool.isort] profile = "black" diff --git a/python/cucim/src/cucim/VERSION b/python/cucim/src/cucim/VERSION new file mode 120000 index 000000000..a4e948506 --- /dev/null +++ b/python/cucim/src/cucim/VERSION @@ -0,0 +1 @@ +../../../../VERSION \ No newline at end of file diff --git a/python/cucim/src/cucim/__init__.py b/python/cucim/src/cucim/__init__.py index ed21dc97d..b655754be 100644 --- a/python/cucim/src/cucim/__init__.py +++ b/python/cucim/src/cucim/__init__.py @@ -40,6 +40,8 @@ submodules = [] submod_attrs = {} +from ._version import __git_commit__, __version__ + try: import cupy @@ -49,18 +51,13 @@ pass try: - from .clara import CuImage, __version__, cli + from .clara import CuImage, cli _is_clara_available = True submodules += ["clara"] submod_attrs["clara"] = ["CuImage", "cli"] except ImportError: - from ._version import get_versions - - __version__ = get_versions()["version"] - del get_versions - del _version - + pass import lazy_loader as lazy @@ -68,7 +65,7 @@ def __dir__(): - return __lazy_dir__() + ["__version__", "is_available"] + return __lazy_dir__() + ["__git_commit__", "__version__", "is_available"] def is_available(module_name: str = "") -> bool: diff --git a/python/cucim/src/cucim/__init__.pyi b/python/cucim/src/cucim/__init__.pyi index 8c7b467f9..646c41b65 100644 --- a/python/cucim/src/cucim/__init__.pyi +++ b/python/cucim/src/cucim/__init__.pyi @@ -13,6 +13,8 @@ # limitations under the License. # +from ._version import __git_commit__, __version__ + submodules = [] try: @@ -25,11 +27,15 @@ except ImportError: pass try: - from .clara import CuImage, __version__, cli # noqa: F401 + from .clara import CuImage, cli # noqa: F401 _is_clara_available = True submodules += ["clara"] except ImportError: - __version__ = "23.12.00" + pass -__all__ = submodules + ["__version__", "is_available"] # noqa: F822 +__all__ = submodules + [ # noqa: F822 + "__git_commit__", + "__version__", + "is_available", +] diff --git a/python/cucim/src/cucim/_version.py b/python/cucim/src/cucim/_version.py new file mode 100644 index 000000000..9abb33848 --- /dev/null +++ b/python/cucim/src/cucim/_version.py @@ -0,0 +1,20 @@ +# Copyright (c) 2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files("cucim").joinpath("VERSION").read_text().strip() +) +__git_commit__ = "" diff --git a/python/cucim/src/cucim/clara/__init__.py b/python/cucim/src/cucim/clara/__init__.py index 4c9fea2f6..537d6b064 100644 --- a/python/cucim/src/cucim/clara/__init__.py +++ b/python/cucim/src/cucim/clara/__init__.py @@ -18,15 +18,7 @@ from . import cli, converter # import hidden methods -from ._cucim import ( - CuImage, - DLDataType, - DLDataTypeCode, - __version__, - cache, - filesystem, - io, -) +from ._cucim import CuImage, DLDataType, DLDataTypeCode, cache, filesystem, io __all__ = [ "cli", @@ -37,7 +29,6 @@ "io", "cache", "converter", - "__version__", ] From aba929ea22fb23abc8befbd4ad5fd841b94bc84c Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 11:57:41 -0400 Subject: [PATCH 34/63] fix package_src_dir in build_wheel.sh --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index d88384bd1..abd17ac1c 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,7 +5,7 @@ set -euo pipefail package_name="cucim" package_dir="python/cucim" -package_src_dir="${package_dir}/src" +package_src_dir="${package_dir}/src/${package_name}" CMAKE_BUILD_TYPE="release" From 80225dc02836de43457b6ff8e7512441ea69039d Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 11:58:26 -0400 Subject: [PATCH 35/63] restore python/cucim/VERSION --- python/cucim/VERSION | 1 + 1 file changed, 1 insertion(+) create mode 120000 python/cucim/VERSION diff --git a/python/cucim/VERSION b/python/cucim/VERSION new file mode 120000 index 000000000..558194c5a --- /dev/null +++ b/python/cucim/VERSION @@ -0,0 +1 @@ +../../VERSION \ No newline at end of file From 6ad0883c589e42d4640787db27f2b42974ce47f6 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 12:22:28 -0400 Subject: [PATCH 36/63] change python/cucim/VERSION used by CMake back to prior method --- ci/release/update-version.sh | 6 ++++-- python/cucim/VERSION | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) mode change 120000 => 100644 python/cucim/VERSION diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index b41c38643..a59a14522 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -36,8 +36,10 @@ sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf. # Centralized version file update echo "${NEXT_FULL_TAG}" > VERSION -echo "${NEXT_FULL_TAG}" > cpp/plugins/cucim.kit.cuslide/VERSION -echo "${NEXT_FULL_TAG}" > cpp/plugins/cucim.kit.cumed/VERSION +# update VERSION files used by C++/CMake (TODO: avoid these duplicate copies) +sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" python/cucim/VERSION +sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cuslide/VERSION +sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cumed/VERSION sed_runner "s#\[Version ${CURRENT_LONG_TAG}\](release_notes/v${CURRENT_LONG_TAG}.md)#\[Version ${NEXT_FULL_TAG}\](release_notes/v${NEXT_FULL_TAG}.md)#g" python/cucim/docs/index.md sed_runner "s/v${CURRENT_LONG_TAG}/v${NEXT_FULL_TAG}/g" python/cucim/docs/getting_started/index.md diff --git a/python/cucim/VERSION b/python/cucim/VERSION deleted file mode 120000 index 558194c5a..000000000 --- a/python/cucim/VERSION +++ /dev/null @@ -1 +0,0 @@ -../../VERSION \ No newline at end of file diff --git a/python/cucim/VERSION b/python/cucim/VERSION new file mode 100644 index 000000000..a193fff41 --- /dev/null +++ b/python/cucim/VERSION @@ -0,0 +1 @@ +23.12.00 From 8937ab107800d11560e68a448ce78793cce5d14e Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 12:53:39 -0400 Subject: [PATCH 37/63] revert to old version folder C++ VERSION files --- ci/release/update-version.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index a59a14522..f6da0e69f 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -21,6 +21,7 @@ CURRENT_LONG_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR}.${CURRENT_PATCH} #Get . for next version NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}') NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}') +NEXT_PATCH=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[3]}') NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" @@ -37,9 +38,10 @@ sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf. # Centralized version file update echo "${NEXT_FULL_TAG}" > VERSION # update VERSION files used by C++/CMake (TODO: avoid these duplicate copies) -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" python/cucim/VERSION -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cuslide/VERSION -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_FULL_TAG}/g" cpp/plugins/cucim.kit.cumed/VERSION +NEXT_CPP_LONG_TAG=${NEXT_MAJOR}.${NEXT_MINOR}.${NEXT_PATCH} # no leading 'v' or trailing 'a' +sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_CPP_LONG_TAG}/g" python/cucim/VERSION +sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_CPP_LONG_TAG}/g" cpp/plugins/cucim.kit.cuslide/VERSION +sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_CPP_LONG_TAG}/g" cpp/plugins/cucim.kit.cumed/VERSION sed_runner "s#\[Version ${CURRENT_LONG_TAG}\](release_notes/v${CURRENT_LONG_TAG}.md)#\[Version ${NEXT_FULL_TAG}\](release_notes/v${NEXT_FULL_TAG}.md)#g" python/cucim/docs/index.md sed_runner "s/v${CURRENT_LONG_TAG}/v${NEXT_FULL_TAG}/g" python/cucim/docs/getting_started/index.md From 9bb33abe96e5e2868bc3df5406fd555350c2d7e8 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 13:36:11 -0400 Subject: [PATCH 38/63] temporarily use separate VERSION_CPP for CMake --- CMakeLists.txt | 2 +- VERSION_CPP | 1 + ci/build_python.sh | 4 ++++ ci/build_wheel.sh | 4 ++++ ci/release/update-version.sh | 1 + 5 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 VERSION_CPP diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ab2e178c..837ec0945 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ cmake_minimum_required(VERSION 3.18) # Set VERSION and BUILD unset(VERSION CACHE) -file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION) +file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION_CPP VERSION) set(PROJECT_VERSION_BUILD dev) # Append local cmake module path diff --git a/VERSION_CPP b/VERSION_CPP new file mode 100644 index 000000000..a193fff41 --- /dev/null +++ b/VERSION_CPP @@ -0,0 +1 @@ +23.12.00 diff --git a/ci/build_python.sh b/ci/build_python.sh index 1597d05fb..74b14abed 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -13,9 +13,13 @@ package_name="cucim" package_dir="python/cucim/src" version=$(rapids-generate-version) +# for CMake VERSION need to truncate any trailing 'a' +version_cpp=${version%a*} + commit=$(git rev-parse HEAD) echo "${version}" > VERSION +echo "${version_cpp}" > VERSION_CPP sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" rapids-logger "Begin py build" diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index abd17ac1c..d172b680e 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -15,6 +15,9 @@ source rapids-date-string version=$(rapids-generate-version) commit=$(git rev-parse HEAD) +# for CMake VERSION_CPP need to truncate any trailing 'a' +version_cpp=${version%a*} + RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Patch project metadata files to include the CUDA version suffix and version override. @@ -24,6 +27,7 @@ PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" # update package name to have the cuda suffix sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} echo "${version}" > VERSION +echo "${version_cpp}" > VERSION_CPP sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_src_dir}/_version.py" if [[ ${PACKAGE_CUDA_SUFFIX} == "-cu12" ]]; then diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index f6da0e69f..133cfac3b 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -39,6 +39,7 @@ sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf. echo "${NEXT_FULL_TAG}" > VERSION # update VERSION files used by C++/CMake (TODO: avoid these duplicate copies) NEXT_CPP_LONG_TAG=${NEXT_MAJOR}.${NEXT_MINOR}.${NEXT_PATCH} # no leading 'v' or trailing 'a' +echo "${NEXT_CPP_LONG_TAG}" > VERSION_CPP sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_CPP_LONG_TAG}/g" python/cucim/VERSION sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_CPP_LONG_TAG}/g" cpp/plugins/cucim.kit.cuslide/VERSION sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_CPP_LONG_TAG}/g" cpp/plugins/cucim.kit.cumed/VERSION From 75cd8e8159d4ee0a941c570ad72dfa3270c4f634 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 14:08:37 -0400 Subject: [PATCH 39/63] version in python/cucim/VERSION also needs to be the C++ version without 'a' --- ci/build_python.sh | 1 + ci/build_wheel.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/ci/build_python.sh b/ci/build_python.sh index 74b14abed..52bee7c9e 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -19,6 +19,7 @@ version_cpp=${version%a*} commit=$(git rev-parse HEAD) echo "${version}" > VERSION +echo "${version_cpp}" > "${package_dir}/VERSION" echo "${version_cpp}" > VERSION_CPP sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index d172b680e..88d38b38a 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -28,6 +28,7 @@ PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} echo "${version}" > VERSION echo "${version_cpp}" > VERSION_CPP +echo "${version_cpp}" > "${package_dir}/VERSION" sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_src_dir}/_version.py" if [[ ${PACKAGE_CUDA_SUFFIX} == "-cu12" ]]; then From 6a4518d635eb72182562b9e7587cc80112c6439f Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 14:34:21 -0400 Subject: [PATCH 40/63] reduce number of separate VERSION files being used by CMake (CMakeLists.txt now use top-level VERSION_CPP) --- ci/build_wheel.sh | 4 ++++ ci/release/update-version.sh | 3 --- cpp/plugins/cucim.kit.cumed/CMakeLists.txt | 2 +- cpp/plugins/cucim.kit.cumed/VERSION | 1 - cpp/plugins/cucim.kit.cuslide/CMakeLists.txt | 2 +- cpp/plugins/cucim.kit.cuslide/VERSION | 1 - python/CMakeLists.txt | 2 +- python/cucim/VERSION | 1 - 8 files changed, 7 insertions(+), 9 deletions(-) delete mode 100644 cpp/plugins/cucim.kit.cumed/VERSION delete mode 100644 cpp/plugins/cucim.kit.cuslide/VERSION delete mode 100644 python/cucim/VERSION diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 88d38b38a..9723ecaa8 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -57,6 +57,7 @@ else fi # First build the C++ lib using CMake via the run script +echo "libcucim version: `cat VERSION_CPP`" ./run build_local libcucim ${CMAKE_BUILD_TYPE} # Current `pip wheel` build assumes all shared libraries exist in this path @@ -64,12 +65,15 @@ PYTHON_CLARA_SO_PREFIX_PATH="./python/cucim/src/cucim/clara/" # Build the C++ cuslide and cumed plugins # (omit copying binaries as they don't go in the wheel) +echo "cuslide version: `cat cpp/plugins/cucim.kit.cuslide/VERSION`" ./run build_local cuslide ${CMAKE_BUILD_TYPE} cp -P -r cpp/plugins/cucim.kit.cuslide/install/lib/* ${PYTHON_CLARA_SO_PREFIX_PATH} +echo "cumed version: `cat cpp/plugins/cucim.kit.cumed/VERSION`" ./run build_local cumed ${CMAKE_BUILD_TYPE} cp -P -r cpp/plugins/cucim.kit.cumed/install/lib/* ${PYTHON_CLARA_SO_PREFIX_PATH} # Compile the Python bindings +echo "cucim bindings version: `cat VERSION_CPP`" ./run build_local cucim ${CMAKE_BUILD_TYPE} cp -P install/lib/*.so ${PYTHON_CLARA_SO_PREFIX_PATH} diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 133cfac3b..cdd32d466 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -40,9 +40,6 @@ echo "${NEXT_FULL_TAG}" > VERSION # update VERSION files used by C++/CMake (TODO: avoid these duplicate copies) NEXT_CPP_LONG_TAG=${NEXT_MAJOR}.${NEXT_MINOR}.${NEXT_PATCH} # no leading 'v' or trailing 'a' echo "${NEXT_CPP_LONG_TAG}" > VERSION_CPP -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_CPP_LONG_TAG}/g" python/cucim/VERSION -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_CPP_LONG_TAG}/g" cpp/plugins/cucim.kit.cuslide/VERSION -sed_runner "s/${CURRENT_LONG_TAG}/${NEXT_CPP_LONG_TAG}/g" cpp/plugins/cucim.kit.cumed/VERSION sed_runner "s#\[Version ${CURRENT_LONG_TAG}\](release_notes/v${CURRENT_LONG_TAG}.md)#\[Version ${NEXT_FULL_TAG}\](release_notes/v${NEXT_FULL_TAG}.md)#g" python/cucim/docs/index.md sed_runner "s/v${CURRENT_LONG_TAG}/v${NEXT_FULL_TAG}/g" python/cucim/docs/getting_started/index.md diff --git a/cpp/plugins/cucim.kit.cumed/CMakeLists.txt b/cpp/plugins/cucim.kit.cumed/CMakeLists.txt index 4073ecfd5..46f61a34d 100644 --- a/cpp/plugins/cucim.kit.cumed/CMakeLists.txt +++ b/cpp/plugins/cucim.kit.cumed/CMakeLists.txt @@ -23,7 +23,7 @@ cmake_minimum_required(VERSION 3.18) # Set VERSION unset(VERSION CACHE) -file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION) +file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../../../VERSION_CPP VERSION) # Append local cmake module path list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules") diff --git a/cpp/plugins/cucim.kit.cumed/VERSION b/cpp/plugins/cucim.kit.cumed/VERSION deleted file mode 100644 index a193fff41..000000000 --- a/cpp/plugins/cucim.kit.cumed/VERSION +++ /dev/null @@ -1 +0,0 @@ -23.12.00 diff --git a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt index 37dbca4f7..4e89df64d 100644 --- a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt +++ b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt @@ -23,7 +23,7 @@ cmake_minimum_required(VERSION 3.18) # Set VERSION unset(VERSION CACHE) -file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION) +file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../../../VERSION_CPP VERSION) # Append local cmake module path list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules") diff --git a/cpp/plugins/cucim.kit.cuslide/VERSION b/cpp/plugins/cucim.kit.cuslide/VERSION deleted file mode 100644 index a193fff41..000000000 --- a/cpp/plugins/cucim.kit.cuslide/VERSION +++ /dev/null @@ -1 +0,0 @@ -23.12.00 diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 2a3277457..502b70241 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -23,7 +23,7 @@ cmake_minimum_required(VERSION 3.18) # Set VERSION unset(VERSION CACHE) -file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../VERSION VERSION) +file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../VERSION_CPP VERSION) # Append local cmake module path list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules") diff --git a/python/cucim/VERSION b/python/cucim/VERSION deleted file mode 100644 index a193fff41..000000000 --- a/python/cucim/VERSION +++ /dev/null @@ -1 +0,0 @@ -23.12.00 From 82a29a6ca8dc54b93426b6b18925e0fd9c37a492 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 14:38:32 -0400 Subject: [PATCH 41/63] remove outdated debug commands --- ci/build_wheel.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 9723ecaa8..2668cd4f7 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -65,15 +65,12 @@ PYTHON_CLARA_SO_PREFIX_PATH="./python/cucim/src/cucim/clara/" # Build the C++ cuslide and cumed plugins # (omit copying binaries as they don't go in the wheel) -echo "cuslide version: `cat cpp/plugins/cucim.kit.cuslide/VERSION`" ./run build_local cuslide ${CMAKE_BUILD_TYPE} cp -P -r cpp/plugins/cucim.kit.cuslide/install/lib/* ${PYTHON_CLARA_SO_PREFIX_PATH} -echo "cumed version: `cat cpp/plugins/cucim.kit.cumed/VERSION`" ./run build_local cumed ${CMAKE_BUILD_TYPE} cp -P -r cpp/plugins/cucim.kit.cumed/install/lib/* ${PYTHON_CLARA_SO_PREFIX_PATH} # Compile the Python bindings -echo "cucim bindings version: `cat VERSION_CPP`" ./run build_local cucim ${CMAKE_BUILD_TYPE} cp -P install/lib/*.so ${PYTHON_CLARA_SO_PREFIX_PATH} From a76f8580f1e51f29013ea82465f4fdb759e03a6a Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 16:00:09 -0400 Subject: [PATCH 42/63] use same test cases across CUDA versions temporarily disable test_cache_hit_miss test case --- ci/test_wheel.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 2c71c4751..a8dea72bd 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -16,10 +16,6 @@ else # skip `test_converter`` (which requires openslide-python and libopenslide) # (A segfault was observed when using the system libopenslide-dev. # No segfault was seen when testing locally using conda-forge's openslide-python) - if [[ ${RAPIDS_PY_CUDA_SUFFIX} == "cu12" ]]; then - python -m pytest ./python/cucim -k "not test_converter" - else - # temporarily disable all clara tests to check if segfault is avoided - python -m pytest ./python/cucim -k "not clara" - fi + # skip `test_cache_hit_miss` (need to investigate and re-enable this test case) + python -m pytest ./python/cucim -k "not test_converter and not test_cache_hit_miss" fi From 488dfc6589509cd213aa4c9ed992d9042da1dc24 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 16:18:58 -0400 Subject: [PATCH 43/63] fix linting issue --- .../tests/performance/clara/test_read_region_memory_usage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/cucim/tests/performance/clara/test_read_region_memory_usage.py b/python/cucim/tests/performance/clara/test_read_region_memory_usage.py index 95d8eae33..12aacab1a 100644 --- a/python/cucim/tests/performance/clara/test_read_region_memory_usage.py +++ b/python/cucim/tests/performance/clara/test_read_region_memory_usage.py @@ -105,9 +105,10 @@ def test_tiff_iterator(testimg_tiff_stripe_4096x4096_256): See issue gh-598: https://github.com/rapidsai/cucim/issues/598 """ - import numpy as np - import random import os + import random + + import numpy as np import psutil slide_path = testimg_tiff_stripe_4096x4096_256 From 922c5db7b51ab0017a74990af95cb7adb5cda3a5 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 18:22:16 -0400 Subject: [PATCH 44/63] change matplotlib to matplotlib-base --- dependencies.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index 6f9f7f93b..c66b0ccb6 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -252,7 +252,7 @@ dependencies: - pytest>=6.2.4 - tifffile>=2022.7.28 - pooch>=1.6.0 # needed to download scikit-image sample data - - matplotlib + - matplotlib-base - output_types: [conda] packages: - imagecodecs>=2021.6.8 From 44e9847545eca499d2c0041bb3159815241c5bf9 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 18:24:57 -0400 Subject: [PATCH 45/63] don't install imagecodecs/openslide for wheel tests (check if skipping these few tests avoid segfault) --- ci/test_wheel.sh | 3 +-- dependencies.yaml | 13 +++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index a8dea72bd..e7efc67f3 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -16,6 +16,5 @@ else # skip `test_converter`` (which requires openslide-python and libopenslide) # (A segfault was observed when using the system libopenslide-dev. # No segfault was seen when testing locally using conda-forge's openslide-python) - # skip `test_cache_hit_miss` (need to investigate and re-enable this test case) - python -m pytest ./python/cucim -k "not test_converter and not test_cache_hit_miss" + python -m pytest ./python/cucim -k "not test_converter" fi diff --git a/dependencies.yaml b/dependencies.yaml index c66b0ccb6..9aa8ebbf8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -255,7 +255,7 @@ dependencies: - matplotlib-base - output_types: [conda] packages: - - imagecodecs>=2021.6.8 + # - imagecodecs>=2021.6.8 - openslide-python>=1.1.2 - pip - pip: @@ -264,11 +264,12 @@ dependencies: packages: - imagecodecs>=2021.6.8 - openslide-python>=1.1.2 - - output_types: [pyproject] - packages: - # skip packages on arm64 that don't provide a wheel - - imagecodecs>=2021.6.8; platform_machine=='x86_64' - - openslide-python>=1.1.2; platform_machine=='x86_64' + # temporarily remove imagecodecs / openslide-python from wheel tests + # - output_types: [pyproject] + # packages: + # # skip packages on arm64 that don't provide a wheel + # - imagecodecs>=2021.6.8; platform_machine=='x86_64' + # - openslide-python>=1.1.2; platform_machine=='x86_64' - output_types: [requirements, pyproject] packages: - opencv-python-headless>=4.6 From 661c18859a704f7627ef7bcd230f0ed7ee6198c1 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 15:25:38 -0700 Subject: [PATCH 46/63] Update ci/build_wheel.sh Co-authored-by: Bradley Dice --- ci/build_wheel.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 2668cd4f7..d52814b87 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -41,8 +41,12 @@ git diff pip install --upgrade pip -#CMake version in the container is too old, install a new version in the python venv -pip install "cmake>=3.26.4" ninja +# Install pip build dependencies (not yet using pyproject.toml) +rapids-dependency-file-generator \ + --file_key "py_build" \ + --output "requirements" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee build_requirements.txt +pip install -r build_requirements.txt # Building the libjpeg-turbo dependency requires YASM # Also need to install openslide dev libraries on the system From c9e8af38f126833f1ccf363dfdbbb3992ce66401 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 18:30:46 -0400 Subject: [PATCH 47/63] remove outdated/broken test skips. CuPy<9.0 is no longer supported --- python/cucim/src/cucim/skimage/transform/tests/test_warps.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/cucim/src/cucim/skimage/transform/tests/test_warps.py b/python/cucim/src/cucim/skimage/transform/tests/test_warps.py index 22a5aad60..403cc12ea 100644 --- a/python/cucim/src/cucim/skimage/transform/tests/test_warps.py +++ b/python/cucim/src/cucim/skimage/transform/tests/test_warps.py @@ -81,7 +81,6 @@ def shift(xy): assert_array_almost_equal(outx, refx) -@cp.testing.with_requires("cupy>=9.0.0b2") def test_warp_matrix(): x = cp.zeros((5, 5), dtype=cp.float64) x[2, 2] = 1 @@ -116,7 +115,6 @@ def test_warp_nd(): assert_array_almost_equal(outx, refx) -@cp.testing.with_requires("cupy>=9.0.0b2") def test_warp_clip(): x = cp.zeros((5, 5), dtype=cp.float64) x[2, 2] = 1 @@ -573,7 +571,6 @@ def test_warp_identity(): assert cp.all(0 == warped_rgb_img[:, :, 1]) -@cp.testing.with_requires("cupy>=9.0.0b2") def test_warp_coords_example(): image = cp.array(astronaut().astype(cp.float32)) assert 3 == image.shape[2] From 76d858bb984cfa42150bbaef991c0f0456a24818 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 18:35:15 -0400 Subject: [PATCH 48/63] fix bad importorskip checks --- .../src/cucim/skimage/measure/tests/test_blur_effect.py | 8 ++++++-- .../tests/test_masked_phase_cross_correlation.py | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/python/cucim/src/cucim/skimage/measure/tests/test_blur_effect.py b/python/cucim/src/cucim/skimage/measure/tests/test_blur_effect.py index 1fe103b9c..8b4afe337 100644 --- a/python/cucim/src/cucim/skimage/measure/tests/test_blur_effect.py +++ b/python/cucim/src/cucim/skimage/measure/tests/test_blur_effect.py @@ -46,8 +46,12 @@ def test_blur_effect_channel_axis(): def test_blur_effect_3d(): """Test that the blur metric works on a 3D image.""" - cells3d = pytest.importorskip("skimage.data.cells3d") - image_3d = cp.array(cells3d()[:, 1, :, :]) # grab just the nuclei + data = pytest.importorskip("skimage.data") + if not hasattr(data, "cells3d"): + pytest.skip( + "cells3d data not available in this version of scikit-image" + ) + image_3d = cp.array(data.cells3d()[:, 1, :, :]) # grab just the nuclei B0 = blur_effect(image_3d) B1 = blur_effect(gaussian(image_3d, sigma=1)) B2 = blur_effect(gaussian(image_3d, sigma=4)) diff --git a/python/cucim/src/cucim/skimage/registration/tests/test_masked_phase_cross_correlation.py b/python/cucim/src/cucim/skimage/registration/tests/test_masked_phase_cross_correlation.py index e83c638fa..79c0c8255 100644 --- a/python/cucim/src/cucim/skimage/registration/tests/test_masked_phase_cross_correlation.py +++ b/python/cucim/src/cucim/skimage/registration/tests/test_masked_phase_cross_correlation.py @@ -75,8 +75,10 @@ def test_masked_registration_random_masks(): def test_masked_registration_3d_contiguous_mask(): """masked_register_translation should be able to register translations between volumes with contiguous masks.""" - brain = pytest.importorskip("skimage.data.brain") - ref_vol = cp.array(brain()[:, ::2, ::2]) + data = pytest.importorskip("skimage.data") + if not hasattr(data, "brain"): + pytest.skip("brain data not available in this version of scikit-image") + ref_vol = cp.array(data.brain()[:, ::2, ::2]) offset = (1, -5, 10) From ca5059d330ea5d7e32a6c17961677b361fa8de1f Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 18:38:33 -0400 Subject: [PATCH 49/63] run rapids-dependency-file-generator --- conda/environments/all_cuda-118_arch-x86_64.yaml | 3 +-- conda/environments/all_cuda-120_arch-x86_64.yaml | 3 +-- python/cucim/pyproject.toml | 4 +--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 7c5083410..3099090b1 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -15,7 +15,6 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - gcc_linux-64=11.* -- imagecodecs>=2021.6.8 - ipython - jbig - lazy_loader>=0.1 @@ -24,7 +23,7 @@ dependencies: - libnvjpeg-dev=11.6.0.55 - libnvjpeg=11.6.0.55 - libwebp-base -- matplotlib +- matplotlib-base - nbsphinx - ninja - numpy>=1.21.3 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 13f0e01a1..8b269e764 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -16,7 +16,6 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - gcc_linux-64=11.* -- imagecodecs>=2021.6.8 - ipython - jbig - lazy_loader>=0.1 @@ -24,7 +23,7 @@ dependencies: - libnvjpeg-dev - libnvjpeg-static - libwebp-base -- matplotlib +- matplotlib-base - nbsphinx - ninja - numpy>=1.21.3 diff --git a/python/cucim/pyproject.toml b/python/cucim/pyproject.toml index 92a9f6f54..af5a81890 100644 --- a/python/cucim/pyproject.toml +++ b/python/cucim/pyproject.toml @@ -62,10 +62,8 @@ Tracker = "https://github.com/rapidsai/cucim/issues" test = [ "GPUtil>=1.4.0", "click", - "imagecodecs>=2021.6.8; platform_machine=='x86_64'", - "matplotlib", + "matplotlib-base", "opencv-python-headless>=4.6", - "openslide-python>=1.1.2; platform_machine=='x86_64'", "pooch>=1.6.0", "psutil>=5.8.0", "pytest-cov>=2.12.1", From 9cc498713b16a5db0678f84bc567fe856b4e7340 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 18:41:49 -0400 Subject: [PATCH 50/63] update comment --- ci/test_wheel.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index e7efc67f3..2fe2c1fb9 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -13,8 +13,6 @@ python -m pip install $(echo ./dist/cucim*.whl)[test] if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then python ./ci/wheel_smoke_test.py else - # skip `test_converter`` (which requires openslide-python and libopenslide) - # (A segfault was observed when using the system libopenslide-dev. - # No segfault was seen when testing locally using conda-forge's openslide-python) - python -m pytest ./python/cucim -k "not test_converter" + # TODO: revisit enabling imagecodecs package during testing + python -m pytest ./python/cucim fi From 33edec4c98316417d3138ad3065b20dbbe2f2dbe Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 18:47:13 -0400 Subject: [PATCH 51/63] matplotlib-base on conda, matplotlib for requirements/pyproject --- dependencies.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 9aa8ebbf8..95f86da54 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -252,10 +252,10 @@ dependencies: - pytest>=6.2.4 - tifffile>=2022.7.28 - pooch>=1.6.0 # needed to download scikit-image sample data - - matplotlib-base - output_types: [conda] packages: - # - imagecodecs>=2021.6.8 + - imagecodecs>=2021.6.8 + - matplotlib-base - openslide-python>=1.1.2 - pip - pip: @@ -272,5 +272,6 @@ dependencies: # - openslide-python>=1.1.2; platform_machine=='x86_64' - output_types: [requirements, pyproject] packages: + - matplotlib - opencv-python-headless>=4.6 - click From 3daf9addb96d5107ee00a52ddb3e6b9cc82088ab Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Tue, 31 Oct 2023 21:15:45 -0400 Subject: [PATCH 52/63] run rapids-dependency-file-generator --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/all_cuda-120_arch-x86_64.yaml | 1 + python/cucim/pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 3099090b1..c038513af 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -15,6 +15,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - gcc_linux-64=11.* +- imagecodecs>=2021.6.8 - ipython - jbig - lazy_loader>=0.1 diff --git a/conda/environments/all_cuda-120_arch-x86_64.yaml b/conda/environments/all_cuda-120_arch-x86_64.yaml index 8b269e764..18f61bf9f 100644 --- a/conda/environments/all_cuda-120_arch-x86_64.yaml +++ b/conda/environments/all_cuda-120_arch-x86_64.yaml @@ -16,6 +16,7 @@ dependencies: - cupy>=12.0.0 - cxx-compiler - gcc_linux-64=11.* +- imagecodecs>=2021.6.8 - ipython - jbig - lazy_loader>=0.1 diff --git a/python/cucim/pyproject.toml b/python/cucim/pyproject.toml index af5a81890..1136ee877 100644 --- a/python/cucim/pyproject.toml +++ b/python/cucim/pyproject.toml @@ -62,7 +62,7 @@ Tracker = "https://github.com/rapidsai/cucim/issues" test = [ "GPUtil>=1.4.0", "click", - "matplotlib-base", + "matplotlib", "opencv-python-headless>=4.6", "pooch>=1.6.0", "psutil>=5.8.0", From 5d9645e5251065a3be87368a69f735cc21013c30 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Wed, 1 Nov 2023 10:56:09 -0400 Subject: [PATCH 53/63] simplify the build_wheel.sh by using 'run build_local all' --- ci/build_wheel.sh | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index d52814b87..7aec5a4ba 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -62,21 +62,7 @@ fi # First build the C++ lib using CMake via the run script echo "libcucim version: `cat VERSION_CPP`" -./run build_local libcucim ${CMAKE_BUILD_TYPE} - -# Current `pip wheel` build assumes all shared libraries exist in this path -PYTHON_CLARA_SO_PREFIX_PATH="./python/cucim/src/cucim/clara/" - -# Build the C++ cuslide and cumed plugins -# (omit copying binaries as they don't go in the wheel) -./run build_local cuslide ${CMAKE_BUILD_TYPE} -cp -P -r cpp/plugins/cucim.kit.cuslide/install/lib/* ${PYTHON_CLARA_SO_PREFIX_PATH} -./run build_local cumed ${CMAKE_BUILD_TYPE} -cp -P -r cpp/plugins/cucim.kit.cumed/install/lib/* ${PYTHON_CLARA_SO_PREFIX_PATH} - -# Compile the Python bindings -./run build_local cucim ${CMAKE_BUILD_TYPE} -cp -P install/lib/*.so ${PYTHON_CLARA_SO_PREFIX_PATH} +./run build_local all ${CMAKE_BUILD_TYPE} cd "${package_dir}" From 4277ae7e748546b6444be194ea9221b5ca78704d Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Wed, 1 Nov 2023 11:33:58 -0400 Subject: [PATCH 54/63] skip openslide-based test if the package is not installed --- python/cucim/tests/unit/clara/converter/test_converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/cucim/tests/unit/clara/converter/test_converter.py b/python/cucim/tests/unit/clara/converter/test_converter.py index 1ded75824..8198e78f7 100644 --- a/python/cucim/tests/unit/clara/converter/test_converter.py +++ b/python/cucim/tests/unit/clara/converter/test_converter.py @@ -18,8 +18,9 @@ import pytest -# skip if imagecodecs package not available (needed by ImageGenerator utility) +# skip if imagecodecs and openslide packages are not available pytest.importorskip("imagecodecs") +pytest.importorskip("openslide") def test_image_converter_stripe_4096x4096_256_jpeg( From 588cd2416288ceaf6d1902029ca2ec59329133c9 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Wed, 1 Nov 2023 11:42:02 -0400 Subject: [PATCH 55/63] update CONTRIBUTING.md --- CONTRIBUTING.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 16c29fcb6..8879f17ba 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -263,19 +263,21 @@ conda install -y -c ${CONDA_BLD_DIR} -c conda-forge \ ## Building a package (for distribution. Including a wheel package for pip) -**Build** - -You can execute the following command to build a wheel file for pip. +**Wheel Build** +If you are using CUDA 12.x, please update pyproject.toml as follows before building the wheel ```bash -./run build_package +sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/cucim/pyproject.toml ``` +This will switch the CuPy dependency to one based on CUDA 12.x instead of 11.x. -The command would use `./temp` folder as a local build folder and build a distribution package into `dist` folder using [dockcross](https://github.com/dockcross/dockcross)'s manylinux2014 docker image. +The wheel can then be build using. -`./run build_package` will reuse local `./temp` folder to reduce the build time. +```bash +python -m pip wheel python/cucim/ -w dist -vvv --no-deps --disable-pip-version-check +``` -If C++ code or dependent packages are updated so the build is failing somehow, please retry it after deleting the `temp` folder under the repository root. +**Note:** It is possible to build the wheel in this way even without compiling the C++ library first, but in that case the `cucim.clara` module will not be importable. **Install** From fe2ebd9deafab99e1f48900913fbd65eb766386b Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Wed, 1 Nov 2023 19:48:11 -0400 Subject: [PATCH 56/63] apply black globally --- .pre-commit-config.yaml | 1 - ci/wheel_smoke_test.py | 8 ++++---- .../src/cuslide/jpeg2k/gen_color_table.py | 14 ++++++++++---- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0290c0e8f..e12e83ae3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,6 @@ repos: rev: 23.10.1 hooks: - id: black - files: (python|legate)/.* args: ["--config", "python/cucim/pyproject.toml"] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.1.3 diff --git a/ci/wheel_smoke_test.py b/ci/wheel_smoke_test.py index 82580e80e..e86a8888e 100644 --- a/ci/wheel_smoke_test.py +++ b/ci/wheel_smoke_test.py @@ -6,14 +6,14 @@ if __name__ == "__main__": # verify that all top-level modules are available - assert cucim.is_available('clara') - assert cucim.is_available('core') - assert cucim.is_available('skimage') + assert cucim.is_available("clara") + assert cucim.is_available("core") + assert cucim.is_available("skimage") # generate a synthetic image and apply a filter img = cucim.skimage.data.binary_blobs(length=512, n_dim=2) assert isinstance(img, cp.ndarray) - assert img.dtype.kind == 'b' + assert img.dtype.kind == "b" assert img.shape == (512, 512) eroded = cucim.skimage.morphology.binary_erosion( diff --git a/cpp/plugins/cucim.kit.cuslide/src/cuslide/jpeg2k/gen_color_table.py b/cpp/plugins/cucim.kit.cuslide/src/cuslide/jpeg2k/gen_color_table.py index 9f442af7c..9d9de21f3 100644 --- a/cpp/plugins/cucim.kit.cuslide/src/cuslide/jpeg2k/gen_color_table.py +++ b/cpp/plugins/cucim.kit.cuslide/src/cuslide/jpeg2k/gen_color_table.py @@ -177,8 +177,12 @@ def gen_b_cb(): def gen_list(values: list, width: int, align: int = 8): text = [] for i in range(0, len(values), width): - text.append(", ".join(("{:>"+str(align)+"}").format(item) - for item in values[i:i + width])) + text.append( + ", ".join( + ("{:>" + str(align) + "}").format(item) + for item in values[i : i + width] + ) + ) return ",\n ".join(text) @@ -189,14 +193,16 @@ def main(output_file_name: str) -> int: b_cb = gen_list(list(gen_b_cb()), 10, 4) with open(output_file_name, "w") as f: - f.write(TEMPLATE % - {"r_cr": r_cr, "g_cb": g_cb, "g_cr": g_cr, "b_cb": b_cb}) + f.write( + TEMPLATE % {"r_cr": r_cr, "g_cb": g_cb, "g_cr": g_cr, "b_cb": b_cb} + ) return 0 if __name__ == "__main__": import sys + if len(sys.argv) != 2: print("Usage: gen_color_table.py ") sys.exit(1) From 6ebfccf54adae4dfbdd8ec55ca6a111e95287585 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Wed, 1 Nov 2023 19:51:41 -0400 Subject: [PATCH 57/63] cleanup test_python section of dependencies.yaml --- dependencies.yaml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 95f86da54..b7ceeef02 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -260,18 +260,12 @@ dependencies: - pip - pip: - opencv-python-headless>=4.6 - - output_types: [requirements] - packages: - - imagecodecs>=2021.6.8 - - openslide-python>=1.1.2 - # temporarily remove imagecodecs / openslide-python from wheel tests - # - output_types: [pyproject] - # packages: - # # skip packages on arm64 that don't provide a wheel - # - imagecodecs>=2021.6.8; platform_machine=='x86_64' - # - openslide-python>=1.1.2; platform_machine=='x86_64' - output_types: [requirements, pyproject] packages: + # temporarily remove imagecodecs / openslide-python from wheel tests + # # skip packages on arm64 that don't provide a wheel + # - imagecodecs>=2021.6.8; platform_machine=='x86_64' + # - openslide-python>=1.1.2; platform_machine=='x86_64' - matplotlib - opencv-python-headless>=4.6 - click From 01dcff636ca48a2b50856c8cc514b4eb30cd1954 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Wed, 1 Nov 2023 16:52:13 -0700 Subject: [PATCH 58/63] Update CONTRIBUTING.md Co-authored-by: Bradley Dice --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8879f17ba..a05dee929 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -271,7 +271,7 @@ sed -i "s/cupy-cuda11x/cupy-cuda12x/g" python/cucim/pyproject.toml ``` This will switch the CuPy dependency to one based on CUDA 12.x instead of 11.x. -The wheel can then be build using. +The wheel can then be built using: ```bash python -m pip wheel python/cucim/ -w dist -vvv --no-deps --disable-pip-version-check From 20df41fd2f96fe32f827eebc99b1ad3888ba7aed Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 3 Nov 2023 17:09:52 -0400 Subject: [PATCH 59/63] just use NEXT_FULL_TAG directly for VERSION_CPP --- ci/release/update-version.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index cdd32d466..a4351fb9e 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -21,7 +21,6 @@ CURRENT_LONG_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR}.${CURRENT_PATCH} #Get . for next version NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}') NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}') -NEXT_PATCH=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[3]}') NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" @@ -37,9 +36,7 @@ sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf. # Centralized version file update echo "${NEXT_FULL_TAG}" > VERSION -# update VERSION files used by C++/CMake (TODO: avoid these duplicate copies) -NEXT_CPP_LONG_TAG=${NEXT_MAJOR}.${NEXT_MINOR}.${NEXT_PATCH} # no leading 'v' or trailing 'a' -echo "${NEXT_CPP_LONG_TAG}" > VERSION_CPP +echo "${NEXT_FULL_TAG}" > VERSION_CPP sed_runner "s#\[Version ${CURRENT_LONG_TAG}\](release_notes/v${CURRENT_LONG_TAG}.md)#\[Version ${NEXT_FULL_TAG}\](release_notes/v${NEXT_FULL_TAG}.md)#g" python/cucim/docs/index.md sed_runner "s/v${CURRENT_LONG_TAG}/v${NEXT_FULL_TAG}/g" python/cucim/docs/getting_started/index.md From 4ca9eff15615a290cc3374dfbadf4e39cdce4f59 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 3 Nov 2023 14:10:35 -0700 Subject: [PATCH 60/63] Update ci/build_wheel.sh Co-authored-by: jakirkham --- ci/build_wheel.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 7aec5a4ba..8afa74ae9 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -48,17 +48,6 @@ rapids-dependency-file-generator \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee build_requirements.txt pip install -r build_requirements.txt -# Building the libjpeg-turbo dependency requires YASM -# Also need to install openslide dev libraries on the system -if ! command -v apt &> /dev/null -then - echo "apt package manager not found, attempting to use yum" - yum install yasm openslide-devel -y -else - echo "apt package manager was found" - apt update - apt install yasm libopenslide-dev -y -fi # First build the C++ lib using CMake via the run script echo "libcucim version: `cat VERSION_CPP`" From f65b052ec9d1d55ad256944cbcfd70f0c8177f08 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 3 Nov 2023 17:56:20 -0400 Subject: [PATCH 61/63] remove unnecessary 'needs' condition --- .github/workflows/build.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 915fd4c4f..1a028553d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,7 +67,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} wheel-build: - needs: [cpp-build] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-23.12 with: From c8f096827f03c5260e34ebcd065f475f48704c05 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Mon, 6 Nov 2023 14:49:10 -0500 Subject: [PATCH 62/63] minor cleanup to build_wheel.sh --- ci/build_wheel.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 8afa74ae9..650318d46 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -36,11 +36,6 @@ if [[ ${PACKAGE_CUDA_SUFFIX} == "-cu12" ]]; then sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} fi -echo "The package name and/or version was modified in the package source. The git diff is:" -git diff - -pip install --upgrade pip - # Install pip build dependencies (not yet using pyproject.toml) rapids-dependency-file-generator \ --file_key "py_build" \ @@ -48,7 +43,6 @@ rapids-dependency-file-generator \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee build_requirements.txt pip install -r build_requirements.txt - # First build the C++ lib using CMake via the run script echo "libcucim version: `cat VERSION_CPP`" ./run build_local all ${CMAKE_BUILD_TYPE} From c05219ff0b3e4fb61be65fbe584a9fafaac6e695 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Mon, 6 Nov 2023 16:06:10 -0500 Subject: [PATCH 63/63] handle stripping of alpha from VERSION string in CMake to simplify CI scripts --- CMakeLists.txt | 4 +++- VERSION | 2 +- VERSION_CPP | 1 - ci/build_python.sh | 9 +++------ ci/build_wheel.sh | 6 ------ ci/release/update-version.sh | 1 - cpp/plugins/cucim.kit.cumed/CMakeLists.txt | 4 +++- cpp/plugins/cucim.kit.cuslide/CMakeLists.txt | 4 +++- python/CMakeLists.txt | 4 +++- 9 files changed, 16 insertions(+), 19 deletions(-) delete mode 100644 VERSION_CPP diff --git a/CMakeLists.txt b/CMakeLists.txt index 837ec0945..ac1aff40e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,9 @@ cmake_minimum_required(VERSION 3.18) # Set VERSION and BUILD unset(VERSION CACHE) -file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION_CPP VERSION) +file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/VERSION VERSION) +# strip alpha version info +string(REGEX REPLACE "a.*$" "" VERSION ${VERSION}) set(PROJECT_VERSION_BUILD dev) # Append local cmake module path diff --git a/VERSION b/VERSION index a193fff41..bb94af9dd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -23.12.00 +23.12.00a56 diff --git a/VERSION_CPP b/VERSION_CPP deleted file mode 100644 index a193fff41..000000000 --- a/VERSION_CPP +++ /dev/null @@ -1 +0,0 @@ -23.12.00 diff --git a/ci/build_python.sh b/ci/build_python.sh index 52bee7c9e..58d8a8e00 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -10,18 +10,15 @@ export CMAKE_GENERATOR=Ninja rapids-print-env package_name="cucim" -package_dir="python/cucim/src" +package_dir="python/cucim" +package_src_dir="${package_dir}/src/${package_name}" version=$(rapids-generate-version) -# for CMake VERSION need to truncate any trailing 'a' -version_cpp=${version%a*} commit=$(git rev-parse HEAD) echo "${version}" > VERSION -echo "${version_cpp}" > "${package_dir}/VERSION" -echo "${version_cpp}" > VERSION_CPP -sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" +sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_src_dir}/_version.py" rapids-logger "Begin py build" diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 650318d46..a39ebf502 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -15,9 +15,6 @@ source rapids-date-string version=$(rapids-generate-version) commit=$(git rev-parse HEAD) -# for CMake VERSION_CPP need to truncate any trailing 'a' -version_cpp=${version%a*} - RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" # Patch project metadata files to include the CUDA version suffix and version override. @@ -27,8 +24,6 @@ PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" # update package name to have the cuda suffix sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} echo "${version}" > VERSION -echo "${version_cpp}" > VERSION_CPP -echo "${version_cpp}" > "${package_dir}/VERSION" sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_src_dir}/_version.py" if [[ ${PACKAGE_CUDA_SUFFIX} == "-cu12" ]]; then @@ -44,7 +39,6 @@ rapids-dependency-file-generator \ pip install -r build_requirements.txt # First build the C++ lib using CMake via the run script -echo "libcucim version: `cat VERSION_CPP`" ./run build_local all ${CMAKE_BUILD_TYPE} cd "${package_dir}" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index a4351fb9e..3669c2dda 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -36,7 +36,6 @@ sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/source/conf. # Centralized version file update echo "${NEXT_FULL_TAG}" > VERSION -echo "${NEXT_FULL_TAG}" > VERSION_CPP sed_runner "s#\[Version ${CURRENT_LONG_TAG}\](release_notes/v${CURRENT_LONG_TAG}.md)#\[Version ${NEXT_FULL_TAG}\](release_notes/v${NEXT_FULL_TAG}.md)#g" python/cucim/docs/index.md sed_runner "s/v${CURRENT_LONG_TAG}/v${NEXT_FULL_TAG}/g" python/cucim/docs/getting_started/index.md diff --git a/cpp/plugins/cucim.kit.cumed/CMakeLists.txt b/cpp/plugins/cucim.kit.cumed/CMakeLists.txt index 46f61a34d..dc775742d 100644 --- a/cpp/plugins/cucim.kit.cumed/CMakeLists.txt +++ b/cpp/plugins/cucim.kit.cumed/CMakeLists.txt @@ -23,7 +23,9 @@ cmake_minimum_required(VERSION 3.18) # Set VERSION unset(VERSION CACHE) -file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../../../VERSION_CPP VERSION) +file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../../../VERSION VERSION) +# strip alpha version info +string(REGEX REPLACE "a.*$" "" VERSION ${VERSION}) # Append local cmake module path list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules") diff --git a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt index 4e89df64d..047b5ee61 100644 --- a/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt +++ b/cpp/plugins/cucim.kit.cuslide/CMakeLists.txt @@ -23,7 +23,9 @@ cmake_minimum_required(VERSION 3.18) # Set VERSION unset(VERSION CACHE) -file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../../../VERSION_CPP VERSION) +file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../../../VERSION VERSION) +# strip alpha version info +string(REGEX REPLACE "a.*$" "" VERSION ${VERSION}) # Append local cmake module path list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules") diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 502b70241..adbbde7e9 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -23,7 +23,9 @@ cmake_minimum_required(VERSION 3.18) # Set VERSION unset(VERSION CACHE) -file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../VERSION_CPP VERSION) +file(STRINGS ${CMAKE_CURRENT_LIST_DIR}/../VERSION VERSION) +# strip alpha version info +string(REGEX REPLACE "a.*$" "" VERSION ${VERSION}) # Append local cmake module path list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules")