Skip to content

Commit

Permalink
add rapids-wheels-anaconda tool
Browse files Browse the repository at this point in the history
  • Loading branch information
AyodeAwe committed Nov 13, 2023
1 parent b2c5e45 commit 2fcd8f5
Show file tree
Hide file tree
Showing 2 changed files with 56 additions and 0 deletions.
56 changes: 56 additions & 0 deletions tools/rapids-wheels-anaconda
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
#!/bin/bash
# A utility script to upload Python wheel packages to Anaconda repository using anaconda-client.

# Positional Arguments:
# 1) wheel name

set -exou pipefail
source rapids-constants
export RAPIDS_SCRIPT_NAME="rapids-wheels-anaconda"

if [ -z "$1" ]; then
rapids-echo-stderr "Must specify input arguments: WHEEL_NAME"
exit 1
fi
WHEEL_NAME="$1"

WHEEL_SEARCH_KEY="wheel_python_${WHEEL_NAME}"

WHEEL_DIR="./dist"
mkdir -p "${WHEEL_DIR}"

S3_PATH=$(rapids-s3-path)
BUCKET_PREFIX=${S3_PATH/s3:\/\/${RAPIDS_DOWNLOADS_BUCKET}\//} # removes s3://rapids-downloads/ from s3://rapids-downloads/ci/rmm/...

# shellcheck disable=SC2016
WHEEL_TARBALLS=$(
set -eo pipefail;
aws \
--output json \
s3api list-objects \
--bucket "${RAPIDS_DOWNLOADS_BUCKET}" \
--prefix "${BUCKET_PREFIX}" \
--page-size 100 \
--query "Contents[?contains(Key, '${WHEEL_SEARCH_KEY}')].Key" \
| jq -c
)
export WHEEL_TARBALLS

# first untar them all
for OBJ in $(jq -nr 'env.WHEEL_TARBALLS | fromjson | .[]'); do
FILENAME=$(basename "${OBJ}")
S3_URI="${S3_PATH}${FILENAME}"

rapids-echo-stderr "Untarring ${S3_URI} into ${WHEEL_DIR}"
aws s3 cp --only-show-errors "${S3_URI}" - | tar xzf - -C "${WHEEL_DIR}"
done

export RAPIDS_RETRY_SLEEP=180
# shellcheck disable=SC2086
rapids-retry anaconda \
-t "${RAPIDS_CONDA_TOKEN}" \
upload \
--skip-existing \
--no-progress \
"${WHEEL_DIR}"/*.whl
echo ""
Empty file added tools/tools
Empty file.

0 comments on commit 2fcd8f5

Please sign in to comment.