Add shellcheck to pre-commit and fix warnings

rapidsai · Jan 23, 2025 · 67c042d · 67c042d
1 parent bd7c69e
commit 67c042d
Show file tree

Hide file tree

Showing 15 changed files with 61 additions and 52 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -80,6 +80,12 @@ repos:
       hooks:
           - id: rapids-dependency-file-generator
             args: ["--clean"]
+    - repo: https://github.com/shellcheck-py/shellcheck-py
+      rev: v0.10.0.1
+      hooks:
+        - id: shellcheck
+          args: ["--severity=warning"]
+          files: ^ci/
 
 default_language_version:
     python: python3
diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -1,12 +1,13 @@
 #!/bin/bash
-# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+# Copyright (c) 2023-2025, NVIDIA CORPORATION.
 set -euo pipefail
 
 rapids-logger "Create test conda environment"
 . /opt/conda/etc/profile.d/conda.sh
 
 RAPIDS_VERSION="$(rapids-version)"
-export RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)"
+RAPIDS_VERSION_MAJOR_MINOR="$(rapids-version-major-minor)"
+export RAPIDS_VERSION_MAJOR_MINOR
 
 rapids-dependency-file-generator \
   --output conda \
@@ -29,7 +30,8 @@ rapids-mamba-retry install \
   "cuml=${RAPIDS_VERSION}" \
   "libcuml=${RAPIDS_VERSION}"
 
-export RAPIDS_DOCS_DIR="$(mktemp -d)"
+RAPIDS_DOCS_DIR="$(mktemp -d)"
+export RAPIDS_DOCS_DIR
 
 rapids-logger "Build CPP docs"
 pushd cpp

diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2023-2024, NVIDIA CORPORATION.
+# Copyright (c) 2023-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -8,7 +8,7 @@ package_dir="python/cuml"
 source rapids-configure-sccache
 source rapids-date-string
 
-RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")"
 
 rapids-generate-version > ./VERSION
 

diff --git a/ci/checks/black_lists.sh b/ci/checks/black_lists.sh
@@ -1,12 +1,12 @@
 #!/bin/bash
-# Copyright (c) 2019-2023, NVIDIA CORPORATION.
+# Copyright (c) 2019-2025, NVIDIA CORPORATION.
 ##########################################
 # cuML black listed function call Tester #
 ##########################################
 
 # PR_TARGET_BRANCH is set by the CI environment
 
-git checkout --quiet $PR_TARGET_BRANCH
+git checkout --quiet "$PR_TARGET_BRANCH"
 
 # Switch back to tip of PR branch
 git checkout --quiet current-pr-branch
@@ -20,16 +20,16 @@ set +H
 RETVAL=0
 
 for black_listed in cudaDeviceSynchronize cudaMalloc cudaMallocManaged cudaFree cudaMallocHost cudaHostAlloc cudaFreeHost; do
-    TMP=`git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$black_listed" $PR_TARGET_BRANCH | grep '^+' | grep -v '^+++' | grep "$black_listed"`
+    TMP=$(git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$black_listed" "$PR_TARGET_BRANCH" | grep '^+' | grep -v '^+++' | grep "$black_listed")
     if [ "$TMP" != "" ]; then
-        for filename in `git --no-pager diff --ignore-submodules -w --minimal --name-only -S"$black_listed" $PR_TARGET_BRANCH`; do
+        for filename in $(git --no-pager diff --ignore-submodules -w --minimal --name-only -S"$black_listed" "$PR_TARGET_BRANCH"); do
             basefilename=$(basename -- "$filename")
             filext="${basefilename##*.}"
             if [ "$filext" != "md" ] && [ "$filext" != "sh" ]; then
-                TMP2=`git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$black_listed" $PR_TARGET_BRANCH -- $filename | grep '^+' | grep -v '^+++' | grep "$black_listed" | grep -vE "^\+[[:space:]]*/{2,}.*$black_listed"`
+                TMP2=$(git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$black_listed" "$PR_TARGET_BRANCH" -- "$filename" | grep '^+' | grep -v '^+++' | grep "$black_listed" | grep -vE "^\+[[:space:]]*/{2,}.*$black_listed")
                 if [ "$TMP2" != "" ]; then
                     echo "=== ERROR: black listed function call $black_listed added to $filename ==="
-                    git --no-pager diff --ignore-submodules -w --minimal -S"$black_listed" $PR_TARGET_BRANCH -- $filename
+                    git --no-pager diff --ignore-submodules -w --minimal -S"$black_listed" "$PR_TARGET_BRANCH" -- "$filename"
                     echo "=== END ERROR ==="
                     RETVAL=1
                 fi
@@ -39,17 +39,17 @@ for black_listed in cudaDeviceSynchronize cudaMalloc cudaMallocManaged cudaFree
 done
 
 for cond_black_listed in cudaMemcpy cudaMemset; do
-    TMP=`git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$cond_black_listed" $PR_TARGET_BRANCH | grep '^+' | grep -v '^+++' | grep -P "$cond_black_listed(?!Async)"`
+    TMP=$(git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$cond_black_listed" "$PR_TARGET_BRANCH" | grep '^+' | grep -v '^+++' | grep -P "$cond_black_listed(?!Async)")
 
     if [ "$TMP" != "" ]; then
-        for filename in `git --no-pager diff --ignore-submodules -w --minimal --name-only -S"$cond_black_listed" $PR_TARGET_BRANCH`; do
+        for filename in $(git --no-pager diff --ignore-submodules -w --minimal --name-only -S"$cond_black_listed" "$PR_TARGET_BRANCH"); do
             basefilename=$(basename -- "$filename")
             filext="${basefilename##*.}"
             if [ "$filext" != "md" ] && [ "$filext" != "sh" ]; then
-                TMP2=`git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$cond_black_listed" $PR_TARGET_BRANCH -- $filename | grep '^+' | grep -v '^+++' | grep -P "$cond_black_listed(?!Async)" | grep -vE "^\+[[:space:]]*/{2,}.*$cond_black_listed"`
+                TMP2=$(git --no-pager diff --ignore-submodules -w --minimal -U0 -S"$cond_black_listed" "$PR_TARGET_BRANCH" -- "$filename" | grep '^+' | grep -v '^+++' | grep -P "$cond_black_listed(?!Async)" | grep -vE "^\+[[:space:]]*/{2,}.*$cond_black_listed")
                 if [ "$TMP2" != "" ]; then
                     echo "=== ERROR: black listed function call $cond_black_listed added to $filename ==="
-                    git --no-pager diff --ignore-submodules -w --minimal -S"$cond_black_listed" $PR_TARGET_BRANCH -- $filename
+                    git --no-pager diff --ignore-submodules -w --minimal -S"$cond_black_listed" "$PR_TARGET_BRANCH" -- "$filename"
                     echo "=== END ERROR ==="
                     RETVAL=1
                 fi

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2019-2024, NVIDIA CORPORATION.
+# Copyright (c) 2019-2025, NVIDIA CORPORATION.
 ########################
 # cuML Version Updater #
 ########################
@@ -13,14 +13,13 @@ NEXT_FULL_TAG=$1
 
 # Get current version
 CURRENT_TAG=$(git tag --merged HEAD | grep -xE '^v.*' | sort --version-sort | tail -n 1 | tr -d 'v')
-CURRENT_MAJOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[1]}')
-CURRENT_MINOR=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[2]}')
-CURRENT_PATCH=$(echo $CURRENT_TAG | awk '{split($0, a, "."); print a[3]}')
+CURRENT_MAJOR=$(echo "$CURRENT_TAG" | awk '{split($0, a, "."); print a[1]}')
+CURRENT_MINOR=$(echo "$CURRENT_TAG" | awk '{split($0, a, "."); print a[2]}')
 CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR}
 
 # Get <major>.<minor> for next version
-NEXT_MAJOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[1]}')
-NEXT_MINOR=$(echo $NEXT_FULL_TAG | awk '{split($0, a, "."); print a[2]}')
+NEXT_MAJOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[1]}')
+NEXT_MINOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[2]}')
 NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR}
 
 # Need to distutils-normalize the original version
@@ -30,7 +29,7 @@ echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG"
 
 # Inplace sed replace; workaround for Linux and Mac
 function sed_runner() {
-    sed -i.bak ''"$1"'' $2 && rm -f ${2}.bak
+    sed -i.bak ''"$1"'' "$2" && rm -f "${2}".bak
 }
 
 

diff --git a/ci/run_cuml_dask_pytests.sh b/ci/run_cuml_dask_pytests.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
-# Copyright (c) 2024, NVIDIA CORPORATION.
+# Copyright (c) 2024-2025, NVIDIA CORPORATION.
 
 # Support invoking run_cuml_dask_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests/dask
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests/dask || exit
 
 python -m pytest --cache-clear "$@" .
diff --git a/ci/run_cuml_integration_pytests.sh b/ci/run_cuml_integration_pytests.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
-# Copyright (c) 2024, NVIDIA CORPORATION.
+# Copyright (c) 2024-2025, NVIDIA CORPORATION.
 
 # Support invoking run_cuml_singlegpu_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests || exit
 
 python -m pytest -p cudf.pandas --cache-clear --ignore=dask -m "not memleak" "$@" --quick_run .
diff --git a/ci/run_cuml_singlegpu_accel_pytests.sh b/ci/run_cuml_singlegpu_accel_pytests.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
-# Copyright (c) 2024, NVIDIA CORPORATION.
+# Copyright (c) 2024-2025, NVIDIA CORPORATION.
 
 # Support invoking run_cuml_singlegpu_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests/experimental/accel
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests/experimental/accel || exit
 
 python -m pytest -p cuml.experimental.accel --cache-clear "$@" .
diff --git a/ci/run_cuml_singlegpu_memleak_pytests.sh b/ci/run_cuml_singlegpu_memleak_pytests.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
-# Copyright (c) 2024, NVIDIA CORPORATION.
+# Copyright (c) 2024-2025, NVIDIA CORPORATION.
 
 # Support invoking run_cuml_singlegpu_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests || exit
 
 python -m pytest --cache-clear --ignore=dask -m "memleak" "$@" .
diff --git a/ci/run_cuml_singlegpu_pytests.sh b/ci/run_cuml_singlegpu_pytests.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
-# Copyright (c) 2024, NVIDIA CORPORATION.
+# Copyright (c) 2024-2025, NVIDIA CORPORATION.
 
 # Support invoking run_cuml_singlegpu_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/cuml/cuml/tests || exit
 
 python -m pytest --cache-clear --ignore=dask -m "not memleak" "$@" .
diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2022-2024, NVIDIA CORPORATION.
+# Copyright (c) 2022-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -39,4 +39,4 @@ export GTEST_OUTPUT=xml:${RAPIDS_TESTS_DIR}/
 ./ci/run_ctests.sh -j9 && EXITCODE=$? || EXITCODE=$?;
 
 rapids-logger "Test script exiting with value: $EXITCODE"
-exit ${EXITCODE}
+exit "${EXITCODE}"
diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2020-2024, NVIDIA CORPORATION.
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
 set -euo pipefail
 
 . /opt/conda/etc/profile.d/conda.sh
@@ -40,17 +40,19 @@ SKIPNBS="cuml_benchmarks.ipynb"
 NBTEST="$(realpath "$(dirname "$0")/utils/nbtest.sh")"
 
 cd notebooks
+
+# shellcheck disable=SC2044
 for nb in $(find . -name "*.ipynb"); do
     nbBasename=$(basename "${nb}")
     # Skip all NBs that use dask (in the code or even in their name)
-    if ((echo "${nb}" | grep -qi dask) || \
-        (grep -q dask "${nb}")); then
+    if (echo "${nb}" | grep -qi dask) || \
+        (grep -q dask "${nb}"); then
         echo "--------------------------------------------------------------------------------"
         echo "SKIPPING: ${nb} (suspected Dask usage, not currently automatable)"
         echo "--------------------------------------------------------------------------------"
     elif (echo " ${SKIPNBS} " | grep -q " ${nbBasename} "); then
         echo "--------------------------------------------------------------------------------"
-        echo "SKIPPING: "${nb}" (listed in skip list)"
+        echo "SKIPPING: ${nb} (listed in skip list)"
         echo "--------------------------------------------------------------------------------"
     else
         nvidia-smi

diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh
@@ -4,7 +4,7 @@
 set -euo pipefail
 
 mkdir -p ./dist
-RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
+RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")"
 RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist
 
 # echo to expand wildcard before adding `[extra]` requires for pip

diff --git a/ci/utils/nbtest.sh b/ci/utils/nbtest.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2020-2021, NVIDIA CORPORATION.
+# Copyright (c) 2020-2025, NVIDIA CORPORATION.
 
 MAGIC_OVERRIDE_CODE="
 def my_run_line_magic(*args, **kwargs):
@@ -24,30 +24,30 @@ get_ipython().run_cell_magic=my_run_cell_magic
 NO_COLORS=--colors=NoColor
 EXITCODE=0
 NBTMPDIR="$WORKSPACE/tmp"
-mkdir -p ${NBTMPDIR}
+mkdir -p "${NBTMPDIR}"
 
-for nb in $*; do
-    NBFILENAME=$1
+for nb in "$@"; do
+    NBFILENAME=$nb
     NBNAME=${NBFILENAME%.*}
     NBNAME=${NBNAME##*/}
     NBTESTSCRIPT=${NBTMPDIR}/${NBNAME}-test.py
     shift
 
     echo --------------------------------------------------------------------------------
-    echo STARTING: ${NBNAME}
+    echo STARTING: "${NBNAME}"
     echo --------------------------------------------------------------------------------
-    jupyter nbconvert --to script ${NBFILENAME} --output ${NBTMPDIR}/${NBNAME}-test
-    echo "${MAGIC_OVERRIDE_CODE}" > ${NBTMPDIR}/tmpfile
-    cat ${NBTESTSCRIPT} >> ${NBTMPDIR}/tmpfile
-    mv ${NBTMPDIR}/tmpfile ${NBTESTSCRIPT}
+    jupyter nbconvert --to script "${NBFILENAME}" --output "${NBTMPDIR}"/"${NBNAME}"-test
+    echo "${MAGIC_OVERRIDE_CODE}" > "${NBTMPDIR}"/tmpfile
+    cat "${NBTESTSCRIPT}" >> "${NBTMPDIR}"/tmpfile
+    mv "${NBTMPDIR}"/tmpfile "${NBTESTSCRIPT}"
 
     echo "Running \"ipython ${NO_COLORS} ${NBTESTSCRIPT}\" on $(date)"
     echo
     time bash -c "ipython ${NO_COLORS} ${NBTESTSCRIPT}; EC=\$?; echo -------------------------------------------------------------------------------- ; echo DONE: ${NBNAME}; exit \$EC"
     NBEXITCODE=$?
     echo EXIT CODE: ${NBEXITCODE}
     echo
-    EXITCODE=$((EXITCODE | ${NBEXITCODE}))
+    EXITCODE=$((EXITCODE | "${NBEXITCODE}"))
 done
 
 exit ${EXITCODE}
diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2024, NVIDIA CORPORATION.
+# Copyright (c) 2024-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -9,10 +9,10 @@ rapids-logger "validate packages with 'pydistcheck'"
 
 pydistcheck \
     --inspect \
-    "$(echo ${wheel_dir_relative_path}/*.whl)"
+    "$(echo "${wheel_dir_relative_path}"/*.whl)"
 
 rapids-logger "validate packages with 'twine'"
 
 twine check \
     --strict \
-    "$(echo ${wheel_dir_relative_path}/*.whl)"
+    "$(echo "${wheel_dir_relative_path}"/*.whl)"