From dbaebabfb5941e3ec91dbdde39b3ca7ce93ee209 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 16 Nov 2023 20:47:57 -0500 Subject: [PATCH 1/9] install imagecodecs and openslide-python dependencies needed by a small subset of tests --- dependencies.yaml | 7 +++---- python/cucim/pyproject.toml | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index b7ceeef02..3e7732126 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -262,10 +262,9 @@ dependencies: - opencv-python-headless>=4.6 - output_types: [requirements, pyproject] packages: - # temporarily remove imagecodecs / openslide-python from wheel tests - # # skip packages on arm64 that don't provide a wheel - # - imagecodecs>=2021.6.8; platform_machine=='x86_64' - # - openslide-python>=1.1.2; platform_machine=='x86_64' + # skip packages on arm64 that don't provide a wheel + - imagecodecs>=2021.6.8; platform_machine=='x86_64' + - openslide-python>=1.1.2; platform_machine=='x86_64' - matplotlib - opencv-python-headless>=4.6 - click diff --git a/python/cucim/pyproject.toml b/python/cucim/pyproject.toml index 1136ee877..92a9f6f54 100644 --- a/python/cucim/pyproject.toml +++ b/python/cucim/pyproject.toml @@ -62,8 +62,10 @@ Tracker = "https://github.com/rapidsai/cucim/issues" test = [ "GPUtil>=1.4.0", "click", + "imagecodecs>=2021.6.8; platform_machine=='x86_64'", "matplotlib", "opencv-python-headless>=4.6", + "openslide-python>=1.1.2; platform_machine=='x86_64'", "pooch>=1.6.0", "psutil>=5.8.0", "pytest-cov>=2.12.1", From cac9f41656d8ecb28825ee4f9252e27cbb4598b1 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Thu, 16 Nov 2023 23:05:52 -0500 Subject: [PATCH 2/9] try setting LD_LIBRARY_PATH to include libopenslide location --- ci/test_wheel.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 2fe2c1fb9..aff107c62 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -9,6 +9,9 @@ RAPIDS_PY_WHEEL_NAME="cucim_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-fro # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/cucim*.whl)[test] +# append folder containing libopenslide.so.0 to LD_LIBRARY_PATH +export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib/x86_64-linux-gnu" + # Run smoke tests for aarch64 pull requests if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then python ./ci/wheel_smoke_test.py From ccaa73ca0c1823124c861cd413ea8a50e28d3851 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 17 Nov 2023 08:10:49 -0500 Subject: [PATCH 3/9] print openslide location during wheel testing --- ci/test_wheel.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index aff107c62..cbd2b6c14 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -9,13 +9,22 @@ RAPIDS_PY_WHEEL_NAME="cucim_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-fro # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/cucim*.whl)[test] -# append folder containing libopenslide.so.0 to LD_LIBRARY_PATH -export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib/x86_64-linux-gnu" - # Run smoke tests for aarch64 pull requests if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then python ./ci/wheel_smoke_test.py else + # verify if openslide package has been installed + echo `dpkg -l | grep openslide` + + # append folder containing libopenslide.so.0 to LD_LIBRARY_PATH (should not be needed) + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib/$(arch)-linux-gnu" + + # try importing like openslide-python does + echo `python -c "import ctypes; print(ctypes.cdll.LoadLibrary('libopenslide.so.0')); print('\n')"` + + # try importing with full path + echo `python -c "import ctypes; print(ctypes.cdll.LoadLibrary('/usr/lib/x86_64-linux-gnu/libopenslide.so.0')); print('\n')"` + # TODO: revisit enabling imagecodecs package during testing python -m pytest ./python/cucim fi From 9b042e102898c8cf6d5fa72d17f7666c8cdbeb26 Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 17 Nov 2023 18:58:43 -0500 Subject: [PATCH 4/9] install libopenslide-dev on x86_64 --- ci/test_wheel.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index cbd2b6c14..d2dd9e4ca 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -13,6 +13,9 @@ python -m pip install $(echo ./dist/cucim*.whl)[test] if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then python ./ci/wheel_smoke_test.py else + + DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libopenslide-dev + # verify if openslide package has been installed echo `dpkg -l | grep openslide` From 099fb01559f8789f2e688e52e4a1d57ded3071bc Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 17 Nov 2023 16:29:28 -0800 Subject: [PATCH 5/9] Update ci/test_wheel.sh Co-authored-by: jakirkham --- ci/test_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index d2dd9e4ca..0f2bbada0 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -14,7 +14,7 @@ if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then python ./ci/wheel_smoke_test.py else - DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libopenslide-dev + DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libopenslide0 # verify if openslide package has been installed echo `dpkg -l | grep openslide` From b644477ef1aa7277518ef56c86f46a7295b881ac Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 17 Nov 2023 21:32:01 -0500 Subject: [PATCH 6/9] call apt update first --- ci/test_wheel.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 0f2bbada0..ebdace57e 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -14,6 +14,7 @@ if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then python ./ci/wheel_smoke_test.py else + DEBIAN_FRONTEND=noninteractive apt update DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libopenslide0 # verify if openslide package has been installed From 29a5e142660cd97dfa103eb19e328f0d31a7b21f Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Mon, 20 Nov 2023 18:28:58 -0500 Subject: [PATCH 7/9] skip the specific test cases that fail on CI --- ci/test_wheel.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index ebdace57e..721be394b 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -29,6 +29,8 @@ else # try importing with full path echo `python -c "import ctypes; print(ctypes.cdll.LoadLibrary('/usr/lib/x86_64-linux-gnu/libopenslide.so.0')); print('\n')"` - # TODO: revisit enabling imagecodecs package during testing - python -m pytest ./python/cucim + # TODO: fix and remove two test cases skipped here + # test_cache_hit_miss currently fails + # test_converter passes, but a segfault occurs on pytest exit when it is enabled + python -m pytest ./python/cucim -k "not test_converter and not test_cache_hit_miss" fi From 55dbf6e4efc3396b72ab5436163b1690594e3b6b Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Mon, 20 Nov 2023 18:29:46 -0500 Subject: [PATCH 8/9] remove debug/testing lines from the test_wheel.sh script --- ci/test_wheel.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 721be394b..4630c5548 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -17,18 +17,6 @@ else DEBIAN_FRONTEND=noninteractive apt update DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libopenslide0 - # verify if openslide package has been installed - echo `dpkg -l | grep openslide` - - # append folder containing libopenslide.so.0 to LD_LIBRARY_PATH (should not be needed) - export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib/$(arch)-linux-gnu" - - # try importing like openslide-python does - echo `python -c "import ctypes; print(ctypes.cdll.LoadLibrary('libopenslide.so.0')); print('\n')"` - - # try importing with full path - echo `python -c "import ctypes; print(ctypes.cdll.LoadLibrary('/usr/lib/x86_64-linux-gnu/libopenslide.so.0')); print('\n')"` - # TODO: fix and remove two test cases skipped here # test_cache_hit_miss currently fails # test_converter passes, but a segfault occurs on pytest exit when it is enabled From 6a318cd2ff7ea6fbe2d85a3a7c672fa8db9817ed Mon Sep 17 00:00:00 2001 From: Gregory Lee Date: Fri, 1 Dec 2023 17:28:03 -0500 Subject: [PATCH 9/9] skip clara module tests on CUDA 11 only --- ci/test_wheel.sh | 13 +++++++++---- python/cucim/tests/unit/clara/test_image_cache.py | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 4630c5548..81f7f53e5 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -9,6 +9,8 @@ RAPIDS_PY_WHEEL_NAME="cucim_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-fro # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/cucim*.whl)[test] +CUDA_MAJOR_VERSION=${RAPIDS_CUDA_VERSION:0:2} + # Run smoke tests for aarch64 pull requests if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then python ./ci/wheel_smoke_test.py @@ -17,8 +19,11 @@ else DEBIAN_FRONTEND=noninteractive apt update DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends libopenslide0 - # TODO: fix and remove two test cases skipped here - # test_cache_hit_miss currently fails - # test_converter passes, but a segfault occurs on pytest exit when it is enabled - python -m pytest ./python/cucim -k "not test_converter and not test_cache_hit_miss" + if [[ ${CUDA_MAJOR_VERSION} == "11" ]]; then + # Omit I/O-related tests in ./python/cucim/tests due to known CUDA bug + # with dynamic loading of libcufile. + python -m pytest ./python/cucim/src/ + else + python -m pytest ./python/cucim + fi fi diff --git a/python/cucim/tests/unit/clara/test_image_cache.py b/python/cucim/tests/unit/clara/test_image_cache.py index 1b8987765..258e01da4 100644 --- a/python/cucim/tests/unit/clara/test_image_cache.py +++ b/python/cucim/tests/unit/clara/test_image_cache.py @@ -196,6 +196,7 @@ def test_reserve_more_cache_memory(): assert cache.miss_count == 0 +@pytest.mark.skip(reason="currently fails (gh-626)") def test_cache_hit_miss(testimg_tiff_stripe_32x24_16_jpeg): from cucim import CuImage from cucim.clara.cache import preferred_memory_capacity