From 1166f87ba821f03a88522fabbb1ab86f05b50b0f Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 13 Dec 2024 14:16:18 -0500 Subject: [PATCH 01/18] deps(kvikio): add kvikio to CUDA version matrices Adding this now that wheels are available --- dependencies.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 7b4d9434..b234cde8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -188,14 +188,13 @@ dependencies: - numactl-devel-cos7-aarch64 - output_types: [requirements, pyproject] matrices: - # kvikio should be added to the CUDA-version-specific matrices once there are wheels available - # ref: https://github.com/rapidsai/kvikio/pull/369 - matrix: cuda: "12.*" cuda_suffixed: "true" packages: - cudf-cu12==25.2.*,>=0.0.0a0 - dask-cudf-cu12==25.2.*,>=0.0.0a0 + - kvikio-cu12==25.2.*,>=0.0.0a0 - ucx-py-cu12==0.42.*,>=0.0.0a0 - matrix: cuda: "11.*" @@ -203,6 +202,7 @@ dependencies: packages: - cudf-cu11==25.2.*,>=0.0.0a0 - dask-cudf-cu11==25.2.*,>=0.0.0a0 + - kvikio-cu11==25.2.*,>=0.0.0a0 - ucx-py-cu11==0.42.*,>=0.0.0a0 - matrix: packages: From 05bd3a81d90864244760def48fcfacc0b5546dca Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 13 Dec 2024 14:16:52 -0500 Subject: [PATCH 02/18] test(wheels): enable wheel tests in CI --- .github/workflows/pr.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1193161a..0dc9a360 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -53,3 +53,12 @@ jobs: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] script: "ci/build_wheel.sh" + wheel-tests: + needs: wheel-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 + with: + build_type: pull-request + script: "ci/test_wheel.sh" + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] From 07a03c9ad1a96b2173d78fe3cea564b5e9c76ed5 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 13 Dec 2024 16:11:38 -0500 Subject: [PATCH 03/18] chore: add test_wheel script --- ci/test_wheel.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 ci/test_wheel.sh diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh new file mode 100644 index 00000000..74d03b84 --- /dev/null +++ b/ci/test_wheel.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -eou pipefail + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="dask_cuda-${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install $(echo ./dist/dask_cuda*.whl)[test] + +python -m pytest -n 8 ./python/dask_cuda/tests From edf038df79ddb48786008e4d32e9bc689e726aac Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 13 Dec 2024 16:14:47 -0500 Subject: [PATCH 04/18] chore: add wheel-tests to pr builder workflow --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0dc9a360..72bc3d78 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,6 +17,7 @@ jobs: - conda-python-tests - docs-build - wheel-build + - wheel-tests secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.02 checks: From 08d01235f836beb5bb91bccc2a78e75f3a1b92f7 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Fri, 13 Dec 2024 16:40:50 -0500 Subject: [PATCH 05/18] chore: chmod +x --- ci/test_wheel.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/test_wheel.sh diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh old mode 100644 new mode 100755 From 680e1868caa7f9e9f2244c109a8f01a762124c88 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 16 Dec 2024 09:55:11 -0500 Subject: [PATCH 06/18] chore: fix wheel name generation We want a name like: `dask_cuda_wheel_python_dask_cuda_py312_x86_64.tar.gz` from these environment variables --- ci/test_wheel.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 74d03b84..2048a77e 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -3,8 +3,7 @@ set -eou pipefail -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="dask_cuda-${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_VERSION="312" RAPIDS_PY_WHEEL_NAME="dask-cuda" rapids-download-wheels-from-s3 ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/dask_cuda*.whl)[test] From b750b4713d0ac3f8b8c8f45cfa54aa7ae48c93da Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 6 Jan 2025 09:31:48 -0500 Subject: [PATCH 07/18] Apply suggestions from code review Co-authored-by: James Lamb --- .github/workflows/pr.yaml | 2 -- ci/test_wheel.sh | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 72bc3d78..ef24aab8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -61,5 +61,3 @@ jobs: with: build_type: pull-request script: "ci/test_wheel.sh" - # This selects "ARCH=amd64 + the latest supported Python + CUDA". - matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 2048a77e..e7c35f57 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -1,9 +1,9 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. set -eou pipefail -RAPIDS_PY_VERSION="312" RAPIDS_PY_WHEEL_NAME="dask-cuda" rapids-download-wheels-from-s3 ./dist +RAPIDS_PY_WHEEL_NAME="dask-cuda" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/dask_cuda*.whl)[test] From 0a43f3868f26712e32dcc0777234d81bdf801aff Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 6 Jan 2025 09:32:16 -0500 Subject: [PATCH 08/18] chore: bump copyright to year PR will land --- ci/test_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index e7c35f57..17fe660b 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. +# Copyright (c) 2025, NVIDIA CORPORATION. set -eou pipefail From 30fddec2635532b3c52dd9bc5d39650938df9cd5 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 6 Jan 2025 10:10:54 -0500 Subject: [PATCH 09/18] chore: build wheels for each arch to be tested --- .github/workflows/pr.yaml | 3 --- ci/test_wheel.sh | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ef24aab8..4db64d7b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -50,9 +50,6 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: build_type: pull-request - # Package is pure Python and only ever requires one build. - # This selects "ARCH=amd64 + the latest supported Python + CUDA". - matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] script: "ci/build_wheel.sh" wheel-tests: needs: wheel-build diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 17fe660b..ba8f3a7e 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -3,7 +3,7 @@ set -eou pipefail -RAPIDS_PY_WHEEL_NAME="dask-cuda" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="dask-cuda" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/dask_cuda*.whl)[test] From 1e20b80fd938e2de9b27d5619fce410bc06b5c6b Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 6 Jan 2025 10:13:53 -0500 Subject: [PATCH 10/18] revert: don't build a bunch of pure wheels --- .github/workflows/pr.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 4db64d7b..ef24aab8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -50,6 +50,9 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.02 with: build_type: pull-request + # Package is pure Python and only ever requires one build. + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]) | [.] script: "ci/build_wheel.sh" wheel-tests: needs: wheel-build From d11ed1786046b919ef1f92e749f9dc900b98aacf Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 6 Jan 2025 10:15:26 -0500 Subject: [PATCH 11/18] ci: set PY_WHEEL_PURE in build_wheel so it's discoverable in the tests --- ci/build_wheel.sh | 2 +- ci/test_wheel.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 760e46e3..741aeab9 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -10,4 +10,4 @@ rapids-generate-version > ./VERSION python -m pip wheel . -w dist -v --no-deps --disable-pip-version-check ./ci/validate_wheel.sh dist -RAPIDS_PY_WHEEL_NAME="dask-cuda" rapids-upload-wheels-to-s3 dist +RAPIDS_PY_WHEEL_NAME="dask-cuda" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 dist diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index ba8f3a7e..17fe660b 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -3,7 +3,7 @@ set -eou pipefail -RAPIDS_PY_WHEEL_NAME="dask-cuda" rapids-download-wheels-from-s3 python ./dist +RAPIDS_PY_WHEEL_NAME="dask-cuda" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/dask_cuda*.whl)[test] From 3689da8ba6e632bab5641457e177d1189731c602 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 6 Jan 2025 11:09:46 -0500 Subject: [PATCH 12/18] remove invalid [test] extra --- ci/test_wheel.sh | 2 +- dependencies.yaml | 8 -------- pyproject.toml | 8 -------- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 17fe660b..cab01055 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -6,6 +6,6 @@ set -eou pipefail RAPIDS_PY_WHEEL_NAME="dask-cuda" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/dask_cuda*.whl)[test] +python -m pip install $(echo ./dist/dask_cuda*.whl) python -m pytest -n 8 ./python/dask_cuda/tests diff --git a/dependencies.yaml b/dependencies.yaml index b234cde8..4b2fe2a0 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -47,14 +47,6 @@ files: table: project includes: - run_python - py_test: - output: pyproject - pyproject_dir: . - extras: - table: project.optional-dependencies - key: test - includes: - - test_python py_docs: output: pyproject pyproject_dir: . diff --git a/pyproject.toml b/pyproject.toml index 7469a345..f1d67c2a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,14 +49,6 @@ docs = [ "sphinx-click>=2.7.1", "sphinx-rtd-theme>=0.5.1", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. -test = [ - "cudf==25.2.*,>=0.0.0a0", - "dask-cudf==25.2.*,>=0.0.0a0", - "kvikio==25.2.*,>=0.0.0a0", - "pytest", - "pytest-cov", - "ucx-py==0.42.*,>=0.0.0a0", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] Homepage = "https://github.com/rapidsai/dask-cuda" From 058ed44308dc4638cc367c10a20dc7d1c3eb9db3 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 6 Jan 2025 11:23:49 -0500 Subject: [PATCH 13/18] generate test dependencies and install in test_wheel.sh --- ci/test_wheel.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index cab01055..4aad0d69 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -5,6 +5,16 @@ set -eou pipefail RAPIDS_PY_WHEEL_NAME="dask-cuda" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist +# Install cuda-suffixed dependencies b/c while `dask-cuda` has no cuda suffix, the test dependencies do +rapids-dependency-file-generator \ + --output requirements \ + --file-key "test_python" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ +| tee /tmp/requirements-test.txt + +rapids-logger "Installing test dependencies" +python -m pip install -v --prefer-binary -r /tmp/requirements-test.txt + # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/dask_cuda*.whl) From a44356a6620d78ce2e7b1c9b4017ded91c1b100a Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 6 Jan 2025 12:02:15 -0500 Subject: [PATCH 14/18] don't use pytest-xdist for now --- ci/test_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 4aad0d69..fc39f5b1 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -18,4 +18,4 @@ python -m pip install -v --prefer-binary -r /tmp/requirements-test.txt # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install $(echo ./dist/dask_cuda*.whl) -python -m pytest -n 8 ./python/dask_cuda/tests +python -m pytest ./python/dask_cuda/tests From 14177a8555d29319920907ab23fabd91de02071e Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 6 Jan 2025 12:05:35 -0500 Subject: [PATCH 15/18] add wheel tests to workflows --- .github/workflows/test.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 237a38a0..c6e2e852 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,3 +22,12 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} + wheel-tests: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-25.02 + with: + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/test_wheel.sh From e410d1d91bc8ffd872d458736a6b597109fcfb6c Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 6 Jan 2025 12:25:17 -0500 Subject: [PATCH 16/18] combine dep install and wheel install --- ci/test_wheel.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index fc39f5b1..cfd48716 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -13,9 +13,7 @@ rapids-dependency-file-generator \ | tee /tmp/requirements-test.txt rapids-logger "Installing test dependencies" -python -m pip install -v --prefer-binary -r /tmp/requirements-test.txt - -# echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/dask_cuda*.whl) +# echo to expand wildcard +python -m pip install -v --prefer-binary -r /tmp/requirements-test.txt $(echo ./dist/dask_cuda*.whl) python -m pytest ./python/dask_cuda/tests From 22a2a69a417f9ccba978d1bf1cc2a4a010d59b37 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 6 Jan 2025 12:25:51 -0500 Subject: [PATCH 17/18] point at the actual test directory --- ci/test_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index cfd48716..b92458fb 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -16,4 +16,4 @@ rapids-logger "Installing test dependencies" # echo to expand wildcard python -m pip install -v --prefer-binary -r /tmp/requirements-test.txt $(echo ./dist/dask_cuda*.whl) -python -m pytest ./python/dask_cuda/tests +python -m pytest ./dask_cuda/tests From f5ff692dbf34eb74263d0afb17334238b2362c28 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 6 Jan 2025 13:13:29 -0500 Subject: [PATCH 18/18] skip ucxx tests --- ci/test_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index b92458fb..f883a735 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -16,4 +16,4 @@ rapids-logger "Installing test dependencies" # echo to expand wildcard python -m pip install -v --prefer-binary -r /tmp/requirements-test.txt $(echo ./dist/dask_cuda*.whl) -python -m pytest ./dask_cuda/tests +python -m pytest ./dask_cuda/tests -k "not ucxx"