Backport sccache v0.7.7 update to branch-24.02 (#231)

* Update to sccache v0.7.7 (#228) * Fix `devcontainer-utils-vault-s3-init` for `[email protected]` (#229)
rapidsai · Feb 9, 2024 · 1b85460 · 1b85460
1 parent b900eba
commit 1b85460
Show file tree

Hide file tree

Showing 16 changed files with 77 additions and 168 deletions.
diff --git a/.github/actions/build-and-test-feature/action.yml b/.github/actions/build-and-test-feature/action.yml
@@ -8,8 +8,6 @@ inputs:
   vault_host: {type: string, defaut: '', required: false}
   rw_sccache_bucket: {type: string, defaut: '', required: false}
   rw_sccache_region: {type: string, defaut: '', required: false}
-  ro_sccache_bucket: {type: string, defaut: '', required: false}
-  ro_sccache_region: {type: string, defaut: '', required: false}
 
 runs:
   using: composite
@@ -34,5 +32,3 @@ runs:
         vault_host: "${{ inputs.vault_host }}"
         rw_sccache_bucket: "${{ inputs.rw_sccache_bucket }}"
         rw_sccache_region: "${{ inputs.rw_sccache_region }}"
-        ro_sccache_bucket: "${{ inputs.ro_sccache_bucket }}"
-        ro_sccache_region: "${{ inputs.ro_sccache_region }}"
diff --git a/.github/workflows/build-and-test-feature.yml b/.github/workflows/build-and-test-feature.yml
@@ -39,5 +39,3 @@ jobs:
           vault_host: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'https://vault.ops.k8s.rapids.ai' || '' }}"
           rw_sccache_bucket: "${{ secrets.GIST_REPO_READ_ORG_GITHUB_TOKEN && 'rapids-sccache-devs' || '' }}"
           rw_sccache_region: "${{ vars.AWS_REGION }}"
-          ro_sccache_bucket: rapids-sccache-east
-          ro_sccache_region: "${{ vars.AWS_REGION }}"
diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json
@@ -1,7 +1,7 @@
 {
   "name": "NVIDIA RAPIDS devcontainer build utilities",
   "id": "rapids-build-utils",
-  "version": "24.2.7",
+  "version": "24.2.8",
   "description": "A feature to install the RAPIDS devcontainer build utilities",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

diff --git a/features/src/rapids-build-utils/install.sh b/features/src/rapids-build-utils/install.sh
@@ -11,15 +11,16 @@ check_packages bc jq sudo wget gettext-base bash-completion ca-certificates;
 
 # Install yq if not installed
 if ! type yq >/dev/null 2>&1; then
-    YQ_VERSION=latest;
-    find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq;
-
     YQ_BINARY="yq";
     YQ_BINARY+="_$(uname -s | tr '[:upper:]' '[:lower:]')";
     YQ_BINARY+="_${TARGETARCH:-$(dpkg --print-architecture | awk -F'-' '{print $NF}')}";
 
-    wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" \
-        | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/";
+    YQ_VERSION=latest;
+    find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq;
+    while ! wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/"; do
+        echo "(!) YQ version ${YQ_VERSION} failed to download. Attempting to fall back one version to retry...";
+        find_prev_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq;
+    done
 fi
 
 # Install the rapids dependency file generator and conda-merge

diff --git a/features/src/sccache/devcontainer-feature.json b/features/src/sccache/devcontainer-feature.json
@@ -1,14 +1,14 @@
 {
   "name": "sccache",
   "id": "sccache",
-  "version": "24.2.1",
+  "version": "24.2.2",
   "description": "A feature to install sccache",
   "options": {
     "version": {
       "type": "string",
       "proposals": [
         "latest",
-        "0.7.4",
+        "0.7.7",
         "0.4.0",
         "0.3.1",
         "0.3.0",

diff --git a/features/src/utils/devcontainer-feature.json b/features/src/utils/devcontainer-feature.json
@@ -1,7 +1,7 @@
 {
   "name": "devcontainer-utils",
   "id": "utils",
-  "version": "24.2.7",
+  "version": "24.2.8",
   "description": "A feature to install RAPIDS devcontainer utility scripts",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

diff --git a/features/src/utils/install.sh b/features/src/utils/install.sh
@@ -32,15 +32,16 @@ fi
 
 # Install yq if not installed
 if ! type yq >/dev/null 2>&1; then
-    YQ_VERSION=latest;
-    find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq;
-
     YQ_BINARY="yq";
     YQ_BINARY+="_$(uname -s | tr '[:upper:]' '[:lower:]')";
     YQ_BINARY+="_${TARGETARCH:-$(dpkg --print-architecture | awk -F'-' '{print $NF}')}";
 
-    wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" \
-        | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/";
+    YQ_VERSION=latest;
+    find_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq;
+    while ! wget --no-hsts -q -O- "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/${YQ_BINARY}.tar.gz" | tar -C /usr/bin -zf - -x ./${YQ_BINARY} --transform="s/${YQ_BINARY}/yq/"; do
+        echo "(!) YQ version ${YQ_VERSION} failed to download. Attempting to fall back one version to retry...";
+        find_prev_version_from_git_tags YQ_VERSION https://github.com/mikefarah/yq;
+    done
 fi
 
 # Remove built-in anacron configs

diff --git a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/persist.sh b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/persist.sh
@@ -38,7 +38,6 @@ store_s3_creds() {
     local region=;
     local no_bucket=;
     local no_region=;
-    local no_credentials=;
     local aws_access_key_id=;
     local aws_session_token=;
     local aws_secret_access_key=;
@@ -50,7 +49,6 @@ store_s3_creds() {
             region                            |
             no_bucket                         |
             no_region                         |
-            no_credentials                    |
             aws_access_key_id                 |
             aws_session_token                 |
             aws_secret_access_key             |
@@ -64,7 +62,6 @@ store_s3_creds() {
     reset_envvar "AWS_ACCESS_KEY_ID";
     reset_envvar "AWS_SESSION_TOKEN";
     reset_envvar "AWS_SECRET_ACCESS_KEY";
-    reset_envvar "SCCACHE_S3_NO_CREDENTIALS";
 
     mkdir -p ~/.aws;
     rm -f ~/.aws/{config,credentials};
@@ -92,41 +89,37 @@ ________EOF
     fi
 
     if test -f ~/.aws/config; then
-        cat <<________EOF > ~/.aws/config
+        cat <<________EOF > ~/.aws/config2 && mv ~/.aws/config{2,}
 [default]
 $(cat ~/.aws/config)
 ________EOF
     fi
 
-    if ! grep -qE "^$" <<< "${no_credentials:-}"; then
-        export_envvar "SCCACHE_S3_NO_CREDENTIALS" "1";
-    else
 
-        if ! grep -qE "^$" <<< "${aws_access_key_id:-}"; then
-            cat <<____________EOF >> ~/.aws/credentials
+    if ! grep -qE "^$" <<< "${aws_access_key_id:-}"; then
+        cat <<________EOF >> ~/.aws/credentials
 aws_access_key_id=${aws_access_key_id}
-____________EOF
-        fi
+________EOF
+    fi
 
-        if ! grep -qE "^$" <<< "${aws_session_token:-}"; then
-            cat <<____________EOF >> ~/.aws/credentials
+    if ! grep -qE "^$" <<< "${aws_session_token:-}"; then
+        cat <<________EOF >> ~/.aws/credentials
 aws_session_token=${aws_session_token}
-____________EOF
-        fi
+________EOF
+    fi
 
-        if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then
-            cat <<____________EOF >> ~/.aws/credentials
+    if ! grep -qE "^$" <<< "${aws_secret_access_key:-}"; then
+        cat <<________EOF >> ~/.aws/credentials
 aws_secret_access_key=${aws_secret_access_key}
-____________EOF
-        fi
+________EOF
+    fi
 
-        if test -f ~/.aws/credentials; then
-            cat <<____________EOF > ~/.aws/credentials
+    if test -f ~/.aws/credentials; then
+        cat <<________EOF > ~/.aws/credentials2 && mv ~/.aws/credentials{2,}
 [default]
 $(cat ~/.aws/credentials)
-____________EOF
-            chmod 0600 ~/.aws/credentials;
-        fi
+________EOF
+        chmod 0600 ~/.aws/credentials;
     fi
 }
 

diff --git a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/propagate.sh b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/propagate.sh
@@ -16,7 +16,7 @@ ensure_s3_creds_have_propagated() {
 
     while true; do
 
-        if SCCACHE_NO_DAEMON=1 sccache --show-stats >/dev/null 2>&1; then
+        if sccache --start-server >/dev/null 2>&1; then
             if [ "${num_restarts}" -gt "0" ]; then echo "Success!"; fi
             exit 0;
         fi

diff --git a/features/src/utils/opt/devcontainer/bin/vault/s3/creds/test.sh b/features/src/utils/opt/devcontainer/bin/vault/s3/creds/test.sh
@@ -17,47 +17,25 @@ test_aws_creds() {
         fi
     fi
 
-    local bucket="${SCCACHE_BUCKET:-"$(grep 'bucket=' ~/.aws/config 2>/dev/null | sed 's/bucket=//' || echo)"}";
+    local bucket="${SCCACHE_BUCKET:-"$(sed -n 's/bucket=//p' ~/.aws/config 2>/dev/null)"}";
     if [ -z "${bucket:-}" ]; then exit 1; fi
 
-    local region="${SCCACHE_REGION:-"${AWS_DEFAULT_REGION:-"$(grep 'region=' ~/.aws/config 2>/dev/null | sed 's/region=//' || echo)"}"}";
-    local aws_access_key_id="${AWS_ACCESS_KEY_ID:-"$(grep 'aws_access_key_id=' ~/.aws/credentials 2>/dev/null | sed 's/aws_access_key_id=//' || echo)"}";
-    local aws_session_token="${AWS_SESSION_TOKEN:-"$(grep 'aws_session_token=' ~/.aws/credentials 2>/dev/null | sed 's/aws_session_token=//' || echo)"}";
-    local aws_secret_access_key="${AWS_SECRET_ACCESS_KEY:-"$(grep 'aws_secret_access_key=' ~/.aws/credentials 2>/dev/null | sed 's/aws_secret_access_key=//' || echo)"}";
+    local region="${SCCACHE_REGION:-"${AWS_DEFAULT_REGION:-"$(sed -n 's/region=//p' ~/.aws/config 2>/dev/null)"}"}";
+    local aws_access_key_id="${AWS_ACCESS_KEY_ID:-"$(sed -n 's/aws_access_key_id=//p' ~/.aws/credentials 2>/dev/null)"}";
+    local aws_session_token="${AWS_SESSION_TOKEN:-"$(sed -n 's/aws_session_token=//p' ~/.aws/credentials 2>/dev/null)"}";
+    local aws_secret_access_key="${AWS_SECRET_ACCESS_KEY:-"$(sed -n 's/aws_secret_access_key=//p' ~/.aws/credentials 2>/dev/null)"}";
 
     if test -n "$(pgrep sccache || echo)"; then
         sccache --stop-server >/dev/null 2>&1 || true;
     fi
 
-    export SCCACHE_NO_DAEMON="1";
-    export SCCACHE_BUCKET="${bucket:-}";
-    export SCCACHE_REGION="${region:-}";
-    export AWS_ACCESS_KEY_ID="${aws_access_key_id:-}";
-    export AWS_SESSION_TOKEN="${aws_session_token:-}";
-    export AWS_SECRET_ACCESS_KEY="${aws_secret_access_key:-}";
-
-    if ! sccache --show-stats 2>&1 | grep -qE 'Cache location \s+ s3'; then
-
-        export SCCACHE_S3_NO_CREDENTIALS="1";
-
-        export AWS_ACCESS_KEY_ID=;
-        export AWS_SESSION_TOKEN=;
-        export AWS_SECRET_ACCESS_KEY=;
-        export -n AWS_ACCESS_KEY_ID;
-        export -n AWS_SESSION_TOKEN;
-        export -n AWS_SECRET_ACCESS_KEY;
-        unset AWS_ACCESS_KEY_ID;
-        unset AWS_SESSION_TOKEN;
-        unset AWS_SECRET_ACCESS_KEY;
-
-        if sccache --show-stats 2>&1 | grep -qE 'Cache location \s+ s3'; then
-            exit 2;
-        fi
-
-        exit 1;
-    fi
-
-    exit 0;
+    SCCACHE_BUCKET="${bucket:-}" \
+    SCCACHE_REGION="${region:-}" \
+    AWS_ACCESS_KEY_ID="${aws_access_key_id:-}" \
+    AWS_SESSION_TOKEN="${aws_session_token:-}" \
+    AWS_SECRET_ACCESS_KEY="${aws_secret_access_key:-}" \
+    sccache --start-server >/dev/null 2>&1;
+    sccache --show-stats | grep -qE 'Cache location \s+ s3';
 }
 
 if test -n "${devcontainer_utils_debug:-}"; then

diff --git a/features/src/utils/opt/devcontainer/bin/vault/s3/init.sh b/features/src/utils/opt/devcontainer/bin/vault/s3/init.sh
@@ -9,9 +9,9 @@ ____EOF
 
 s3_bucket_auth() {
     cat <<____EOF
-        --aws_access_key_id='$(grep 'aws_access_key_id=' ~/.aws/credentials 2>/dev/null | sed 's/aws_access_key_id=//' || echo)'
-        --aws_session_token='$(grep 'aws_session_token=' ~/.aws/credentials 2>/dev/null | sed 's/aws_session_token=//' || echo)'
-        --aws_secret_access_key='$(grep 'aws_secret_access_key=' ~/.aws/credentials 2>/dev/null | sed 's/aws_secret_access_key=//' || echo)'
+        --aws_access_key_id='$(sed -n 's/aws_access_key_id=//p' ~/.aws/credentials 2>/dev/null)'
+        --aws_session_token='$(sed -n 's/aws_session_token=//p' ~/.aws/credentials 2>/dev/null)'
+        --aws_secret_access_key='$(sed -n 's/aws_secret_access_key=//p' ~/.aws/credentials 2>/dev/null)'
 ____EOF
 }
 
@@ -25,7 +25,7 @@ init_vault_s3_creds() {
             && grep -qE "^$" <<< "${AWS_SECRET_ACCESS_KEY:-}" ; then
                 if test -n "${VAULT_HOST:-}"                  ; then
                     # Generate S3 creds if they don't exist (or are expired)
-                    if devcontainer-utils-vault-s3-creds-test 2>&1 >/dev/null\
+                    if devcontainer-utils-vault-s3-creds-test \
                     || devcontainer-utils-vault-s3-creds-generate; then
                         # Persist creds in ~/.aws dir
                         devcontainer-utils-vault-s3-creds-persist <<< "
@@ -37,34 +37,21 @@ init_vault_s3_creds() {
                     else
                         devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region";
                     fi
+                elif devcontainer-utils-vault-s3-creds-test; then
+                    # bucket is read + write with the current credentials
+                    devcontainer-utils-vault-s3-creds-persist <<< "
+                        $(s3_bucket_args)
+                        $(s3_bucket_auth)
+                    ";
                 else
-                    # If credentials have been mounted in, ensure they're used
-                    case $(devcontainer-utils-vault-s3-creds-test; echo $?) in
-                        # bucket is read + write with the current credentials
-                        [0] )
-                            devcontainer-utils-vault-s3-creds-persist <<< "
-                                $(s3_bucket_args)
-                                $(s3_bucket_auth)
-                            ";;
-                        # bucket is read-only and should be accessed without credentials
-                        [2] )
-                            devcontainer-utils-vault-s3-creds-persist <<< "
-                                --no_credentials
-                                $(s3_bucket_args)
-                            ";;
-                          # bucket is inaccessible
-                          * )
-                            devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region";;
-                    esac
+                    # bucket is inaccessible
+                    devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region";
                 fi
-            elif devcontainer-utils-vault-s3-creds-propagate; then
-                # Block until the new temporary AWS S3 credentials propagate
-                echo -n "";
+            elif ! devcontainer-utils-vault-s3-creds-propagate; then
+                # bucket is inaccessible
+                devcontainer-utils-vault-s3-creds-persist <<< "--no_bucket --no_region";
             fi
         fi
-        . /etc/profile.d/*-devcontainer-utils.sh;
-        # start the sccache server
-        sccache --start-server >/dev/null 2>&1 || true;
     fi
 }
 

diff --git a/features/test/utils/scenarios.json b/features/test/utils/scenarios.json
@@ -7,9 +7,7 @@
       "vault_host": "${localEnv:vault_host}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
-      "rw_sccache_region": "${localEnv:rw_sccache_region}",
-      "ro_sccache_bucket": "${localEnv:ro_sccache_bucket}",
-      "ro_sccache_region": "${localEnv:ro_sccache_region}"
+      "rw_sccache_region": "${localEnv:rw_sccache_region}"
     },
     "features": {
       "ghcr.io/devcontainers/features/common-utils:2": {
@@ -29,7 +27,7 @@
       "cmake": {},
       "ninja": {},
       "sccache": {
-        "version": "0.7.4"
+        "version": "0.7.7"
       },
       "utils": {}
     },
@@ -53,9 +51,7 @@
       "vault_host": "${localEnv:vault_host}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
-      "rw_sccache_region": "${localEnv:rw_sccache_region}",
-      "ro_sccache_bucket": "${localEnv:ro_sccache_bucket}",
-      "ro_sccache_region": "${localEnv:ro_sccache_region}"
+      "rw_sccache_region": "${localEnv:rw_sccache_region}"
     },
     "features": {
       "ghcr.io/devcontainers/features/common-utils:2": {
@@ -75,7 +71,7 @@
       "cmake": {},
       "ninja": {},
       "sccache": {
-        "version": "0.7.4"
+        "version": "0.7.7"
       },
       "utils": {}
     },
@@ -99,9 +95,7 @@
       "vault_host": "${localEnv:vault_host}",
       "VAULT_S3_TTL": "${localEnv:VAULT_S3_TTL}",
       "rw_sccache_bucket": "${localEnv:rw_sccache_bucket}",
-      "rw_sccache_region": "${localEnv:rw_sccache_region}",
-      "ro_sccache_bucket": "${localEnv:ro_sccache_bucket}",
-      "ro_sccache_region": "${localEnv:ro_sccache_region}"
+      "rw_sccache_region": "${localEnv:rw_sccache_region}"
     },
     "features": {
       "ghcr.io/devcontainers/features/common-utils:2": {
@@ -121,7 +115,7 @@
       "cmake": {},
       "ninja": {},
       "sccache": {
-        "version": "0.7.4"
+        "version": "0.7.7"
       },
       "utils": {}
     },