From 8cd4ba212d460ac436ee0d90e765fa3fc06946d4 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 8 Jul 2024 12:47:51 -0500 Subject: [PATCH 1/9] select CUDA-suffixed dependencies for pip env --- .../devcontainer-feature.json | 2 +- .../bin/make-pip-dependencies.sh | 31 ++++++++++++------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index 278d2209..a7803af2 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "24.8.13", + "version": "24.8.14", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 18121243..1588551d 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -59,6 +59,13 @@ make_pip_dependencies() { local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; + # projects that depend on different pip libraries across different CUDA versions + # (e.g. 'cudf' only depending on 'pynvjitlink' from CUDA 12.0 onwards), split up their + # dependency lists with 'cuda_suffixed={true,false}'. + # + # here we want the suffixed versions (like 'pynvjitlink-cu12') + local -r matrix_selectors="arch=$(uname -m);cuda=${cuda_version};py=${python_version};cuda_suffixed=true" + local pip_reqs_txts=(); eval "$(rapids-list-repos "${OPTS[@]}")"; @@ -81,12 +88,12 @@ make_pip_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.${repo_keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_requirements \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output requirements \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done @@ -103,12 +110,12 @@ make_pip_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_requirements \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output requirements \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done From c8bffd41238e6ee960a9cca9b5382c4605228e3a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 22 Jul 2024 07:28:33 -0700 Subject: [PATCH 2/9] change more places --- .../rapids-build-utils/bin/make-pip-dependencies.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 1588551d..e9815871 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -133,12 +133,12 @@ make_pip_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_requirements \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output requirements \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done From 072f5e550be92d83a7fdfda809dbfc86cf69ec84 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 22 Jul 2024 12:15:57 -0500 Subject: [PATCH 3/9] expand comment --- .../opt/rapids-build-utils/bin/make-pip-dependencies.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 1588551d..623a7642 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -59,11 +59,15 @@ make_pip_dependencies() { local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; - # projects that depend on different pip libraries across different CUDA versions + # Projects that depend on different pip libraries across different CUDA versions # (e.g. 'cudf' only depending on 'pynvjitlink' from CUDA 12.0 onwards), split up their # dependency lists with 'cuda_suffixed={true,false}'. # - # here we want the suffixed versions (like 'pynvjitlink-cu12') + # Here we want the suffixed versions (like 'pynvjitlink-cu12'). + # + # It's ok for other RAPIDS libraries to end up in this list (like 'rmm-cu12')... in builds + # where those are also being built in the devcontainer, they'll be filtered out via + # inclusion in the 'pip_noinstall' list below. local -r matrix_selectors="arch=$(uname -m);cuda=${cuda_version};py=${python_version};cuda_suffixed=true" local pip_reqs_txts=(); From 5c93127a2f90e768737fac1d4d0a90e293caf55a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 22 Jul 2024 12:18:03 -0500 Subject: [PATCH 4/9] one more place --- .../rapids-build-utils/bin/make-pip-dependencies.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 623a7642..4b3778fe 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -137,12 +137,12 @@ make_pip_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.requirements.txt"; pip_reqs_txts+=("${file}"); - generate_requirements \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output requirements \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_requirements \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output requirements \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done From 52f567d3d343a5c49bc781db4eef3b413f8c4dba Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 22 Jul 2024 12:19:09 -0500 Subject: [PATCH 5/9] bump version --- features/src/rapids-build-utils/devcontainer-feature.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json index a7803af2..08ae9d90 100644 --- a/features/src/rapids-build-utils/devcontainer-feature.json +++ b/features/src/rapids-build-utils/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "NVIDIA RAPIDS devcontainer build utilities", "id": "rapids-build-utils", - "version": "24.8.14", + "version": "24.8.15", "description": "A feature to install the RAPIDS devcontainer build utilities", "containerEnv": { "BASH_ENV": "/etc/bash.bash_env" From c40f39f09183af6a9421ee02a1104f4dae874613 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 22 Jul 2024 12:18:11 -0700 Subject: [PATCH 6/9] move matrix entry to argument --- .../opt/rapids-build-utils/bin/make-pip-dependencies.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 4b3778fe..eeb50d71 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -13,6 +13,9 @@ # -e,--exclude Path(s) to requirement files of packages to exclude. # -i,--include Path(s) to requirement files of packages to include. # -k,--key Only include the key(s) +# --matrix-entry Matrix entries, in the form 'key=value' to be passed to '--matrix' arg +# of rapids-dependency-file-generator (e.g., 'cuda=12.2'). +# (can be passed multiple times) # @_include_value_options rapids-list-repos -h | tail -n+2 | head -n-3; # --repo Only include dependencies for repo(s). # (default: all repositories) @@ -59,6 +62,8 @@ make_pip_dependencies() { local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; + # Why default to cuda_suffixed=true? + # # Projects that depend on different pip libraries across different CUDA versions # (e.g. 'cudf' only depending on 'pynvjitlink' from CUDA 12.0 onwards), split up their # dependency lists with 'cuda_suffixed={true,false}'. @@ -68,7 +73,8 @@ make_pip_dependencies() { # It's ok for other RAPIDS libraries to end up in this list (like 'rmm-cu12')... in builds # where those are also being built in the devcontainer, they'll be filtered out via # inclusion in the 'pip_noinstall' list below. - local -r matrix_selectors="arch=$(uname -m);cuda=${cuda_version};py=${python_version};cuda_suffixed=true" + test ${#matrix_entry[@]} -eq 0 && matrix_entry=(arch="$(uname -m)" cuda="${cuda_version}" cuda_suffixed=true py="${python_version}"); + local -r matrix_selectors=$(IFS=";"; echo "${matrix_entry[*]}") local pip_reqs_txts=(); From 9e15d707bae678cb5158d1f488f1340c5076f4a1 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 22 Jul 2024 16:50:50 -0700 Subject: [PATCH 7/9] always keep cuda, arch, and py in the rapids-dependency-file-generator --matrix --- .../bin/make-pip-dependencies.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index eeb50d71..8477c342 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -13,8 +13,8 @@ # -e,--exclude Path(s) to requirement files of packages to exclude. # -i,--include Path(s) to requirement files of packages to include. # -k,--key Only include the key(s) -# --matrix-entry Matrix entries, in the form 'key=value' to be passed to '--matrix' arg -# of rapids-dependency-file-generator (e.g., 'cuda=12.2'). +# --matrix-entry Matrix entries, in the form 'key=value' to be added to the '--matrix' arg +# of rapids-dependency-file-generator. # (can be passed multiple times) # @_include_value_options rapids-list-repos -h | tail -n+2 | head -n-3; # --repo Only include dependencies for repo(s). @@ -73,8 +73,18 @@ make_pip_dependencies() { # It's ok for other RAPIDS libraries to end up in this list (like 'rmm-cu12')... in builds # where those are also being built in the devcontainer, they'll be filtered out via # inclusion in the 'pip_noinstall' list below. - test ${#matrix_entry[@]} -eq 0 && matrix_entry=(arch="$(uname -m)" cuda="${cuda_version}" cuda_suffixed=true py="${python_version}"); - local -r matrix_selectors=$(IFS=";"; echo "${matrix_entry[*]}") + local -a _matrix_selectors=( + arch="$(uname -m)" + cuda="${cuda_version}" + cuda_suffixed=true + py="${python_version}" + ); + + # add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win) + local ent; for ent in "${matrix_entry[@]}"; do + _matrix_selectors+=("${ent}") + done + local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}") local pip_reqs_txts=(); From 0df12dfdbe3128a90c15f825bfd520a7e53bf6d3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 23 Jul 2024 08:41:36 -0500 Subject: [PATCH 8/9] Update features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh Co-authored-by: Paul Taylor <178183+trxcllnt@users.noreply.github.com> --- .../opt/rapids-build-utils/bin/make-pip-dependencies.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 8477c342..78be9cf9 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -81,9 +81,7 @@ make_pip_dependencies() { ); # add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win) - local ent; for ent in "${matrix_entry[@]}"; do - _matrix_selectors+=("${ent}") - done + test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}"); local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}") local pip_reqs_txts=(); From 3df408c0f6e0ad300d3493ebe32b948e3c1fac53 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 23 Jul 2024 08:50:17 -0500 Subject: [PATCH 9/9] update make-conda-dependencies too --- .../bin/make-conda-dependencies.sh | 50 ++++++++++++------- .../bin/make-pip-dependencies.sh | 1 + 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh index 99d44806..9087946b 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-dependencies.sh @@ -12,6 +12,9 @@ # -e,--exclude Path(s) to requirement files of packages to exclude. # -i,--include Path(s) to requirement files of packages to include. # -k,--key Only include the key(s) +# --matrix-entry Matrix entries, in the form 'key=value' to be added to the '--matrix' arg +# of rapids-dependency-file-generator. +# (can be passed multiple times) # @_include_value_options rapids-list-repos -h | tail -n+2 | head -n-3; # --repo Only include dependencies for repo(s). # (default: all repositories) @@ -41,6 +44,7 @@ make_conda_dependencies() { test ${#exclude[@]} -eq 0 && exclude=(); test ${#include[@]} -eq 0 && include=(); test ${#key[@]} -eq 0 && key=(all); + test ${#matrix_entry[@]} -eq 0 && matrix_entry=(); local -a _exclude=(); local exc; for exc in "${exclude[@]}"; do @@ -58,6 +62,16 @@ make_conda_dependencies() { local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}"; python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")"; + local -a _matrix_selectors=( + arch="$(uname -m)" + cuda="${cuda_version}" + py="${python_version}" + ); + + # add extra arguments (if there are conflicts, e.g. 'py=3.10;py=3.11', it's fine... the last one will win) + test ${#matrix_entry[@]} -gt 0 && _matrix_selectors+=("${matrix_entry[@]}"); + local -r matrix_selectors=$(IFS=";"; echo "${_matrix_selectors[*]}") + local conda_env_yamls=(); local i; @@ -78,12 +92,12 @@ make_conda_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.${repo_keys[$keyi]}.env.yaml"; conda_env_yamls+=("${file}"); - generate_env_yaml \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output conda \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_env_yaml \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output conda \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done @@ -100,12 +114,12 @@ make_conda_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.lib${!cpp_name}.${repo_keys[$keyi]}.env.yaml"; conda_env_yamls+=("${file}"); - generate_env_yaml \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output conda \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_env_yaml \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output conda \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done @@ -123,12 +137,12 @@ make_conda_dependencies() { for ((keyi=0; keyi < ${#repo_keys[@]}; keyi+=1)); do local file="/tmp/${!repo_name}.${!py_name}.${repo_keys[$keyi]}.env.yaml"; conda_env_yamls+=("${file}"); - generate_env_yaml \ - "${file}" \ - --file-key "${repo_keys[$keyi]}" \ - --output conda \ - --config ~/"${!repo_path}/dependencies.yaml" \ - --matrix "arch=$(uname -m);cuda=${cuda_version};py=${python_version}" \ + generate_env_yaml \ + "${file}" \ + --file-key "${repo_keys[$keyi]}" \ + --output conda \ + --config ~/"${!repo_path}/dependencies.yaml" \ + --matrix "${matrix_selectors}" \ ; done done diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh index 78be9cf9..af4b818d 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-dependencies.sh @@ -44,6 +44,7 @@ make_pip_dependencies() { test ${#exclude[@]} -eq 0 && exclude=(); test ${#include[@]} -eq 0 && include=(); test ${#key[@]} -eq 0 && key=(py_build py_run py_test all); + test ${#matrix_entry[@]} -eq 0 && matrix_entry=(); test ${#requirement[@]} -eq 0 && requirement=(); local -a _exclude=();