Use --qualifier in release manager (elastic#16907)

This commit uses the new --qualifier parameter in the release manager for publishing dra artifacts. Additionally, simplifies the expected variables to rely on a simple `VERSION_QUALIFIER`. Snapshot builds are skipped when VERSION_QUALIFIER is set. Finally, for helping to test DRA PRs, we also allow passing the `DRA_BRANCH` option/env var to override BUILDKITE_BRANCH. Closes elastic/ingest-dev#4856
dliappis · Jan 20, 2025 · 0aca2b8 · 0aca2b8
1 parent 2410483
commit 0aca2b8
Show file tree

Hide file tree

Showing 5 changed files with 111 additions and 102 deletions.
diff --git a/.buildkite/dra_pipeline.yml b/.buildkite/dra_pipeline.yml
@@ -4,8 +4,12 @@ steps:
   - label: ":pipeline: Generate steps"
     command: |
       set -euo pipefail
-
-      echo "--- Building [${WORKFLOW_TYPE}] artifacts"
+      
+      echo "--- Building [$${WORKFLOW_TYPE}] artifacts"
       python3 -m pip install pyyaml
       echo "--- Building dynamic pipeline steps"
-      python3 .buildkite/scripts/dra/generatesteps.py | buildkite-agent pipeline upload
+      python3 .buildkite/scripts/dra/generatesteps.py > steps.yml
+      echo "--- Printing dynamic pipeline steps"
+      cat steps.yml
+      echo "--- Uploading dynamic pipeline steps"
+      cat steps.yml | buildkite-agent pipeline upload
diff --git a/.buildkite/scripts/dra/build_docker.sh b/.buildkite/scripts/dra/build_docker.sh
@@ -9,61 +9,38 @@ source ./$(dirname "$0")/common.sh
 # WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
 case "$WORKFLOW_TYPE" in
     snapshot)
-        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
-        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
-            rake artifact:docker || error "artifact:docker build failed."
-            rake artifact:docker_oss || error "artifact:docker_oss build failed."
-            rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
-            rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
-            if [ "$ARCH" != "aarch64" ]; then
-                rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
-            fi
-        else
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker || error "artifact:docker build failed."
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_oss || error "artifact:docker_oss build failed."
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
-            if [ "$ARCH" != "aarch64" ]; then
-                VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
-            fi
-            # Qualifier is passed from CI as optional field and specify the version postfix
-            # in case of alpha or beta releases:
-            # e.g: 8.0.0-alpha1
-            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
-        fi
-        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
-        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        :  # no-op
         ;;
     staging)
-        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
-        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
-            RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
-            RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
-            RELEASE=1 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
-            RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
-            if [ "$ARCH" != "aarch64" ]; then
-                RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
-            fi
-        else
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker || error "artifact:docker build failed."
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_oss || error "artifact:docker_oss build failed."
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
-            if [ "$ARCH" != "aarch64" ]; then
-                VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
-            fi
-            # Qualifier is passed from CI as optional field and specify the version postfix
-            # in case of alpha or beta releases:
-            # e.g: 8.0.0-alpha1
-            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
-        fi
-        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        export RELEASE=1
         ;;
     *)
         error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
         ;;
 esac
 
+rake artifact:docker || error "artifact:docker build failed."
+rake artifact:docker_oss || error "artifact:docker_oss build failed."
+rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed."
+rake artifact:dockerfiles || error "artifact:dockerfiles build failed."
+
+if [[ "$ARCH" != "aarch64" ]]; then
+  rake artifact:docker_ubi8 || error "artifact:docker_ubi8 build failed."
+fi
+
+if [[ "$WORKFLOW_TYPE" == "staging" ]] && [[ -n "$VERSION_QUALIFIER" ]]; then
+    # Qualifier is passed from CI as optional field and specify the version postfix
+    # in case of alpha or beta releases for staging builds only:
+    # e.g: 8.0.0-alpha1
+    STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER}"
+fi
+
+if [[ "$WORKFLOW_TYPE" == "snapshot" ]]; then
+    STACK_VERSION="${STACK_VERSION}-SNAPSHOT"
+fi
+
+info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+
 info "Saving tar.gz for docker images"
 save_docker_tarballs "${ARCH}" "${STACK_VERSION}"
 

diff --git a/.buildkite/scripts/dra/build_packages.sh b/.buildkite/scripts/dra/build_packages.sh
@@ -7,39 +7,35 @@ echo "####################################################################"
 source ./$(dirname "$0")/common.sh
 
 # WORKFLOW_TYPE is a CI externally configured environment variable that could assume "snapshot" or "staging" values
+info "Building artifacts for the $WORKFLOW_TYPE workflow ..."
+
 case "$WORKFLOW_TYPE" in
     snapshot)
-        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
-        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
-            SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
-        else
-            # Qualifier is passed from CI as optional field and specify the version postfix
-            # in case of alpha or beta releases:
-            # e.g: 8.0.0-alpha1
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
-            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
-        fi
-        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
-        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        :  # no-op
         ;;
     staging)
-        info "Building artifacts for the $WORKFLOW_TYPE workflow..."
-        if [ -z "$VERSION_QUALIFIER_OPT" ]; then
-            RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
-        else
-            # Qualifier is passed from CI as optional field and specify the version postfix
-            # in case of alpha or beta releases:
-            # e.g: 8.0.0-alpha1
-            VERSION_QUALIFIER="$VERSION_QUALIFIER_OPT" RELEASE=1 SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
-            STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
-        fi
-        info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+        export RELEASE=1
         ;;
     *)
         error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
         ;;
 esac
 
+SKIP_DOCKER=1 rake artifact:all || error "rake artifact:all build failed."
+
+if [[ "$WORKFLOW_TYPE" == "staging" ]] && [[ -n "$VERSION_QUALIFIER" ]]; then
+    # Qualifier is passed from CI as optional field and specify the version postfix
+    # in case of alpha or beta releases for staging builds only:
+    # e.g: 8.0.0-alpha1
+    STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER}"
+fi
+
+if [[ "$WORKFLOW_TYPE" == "snapshot" ]]; then
+    STACK_VERSION="${STACK_VERSION}-SNAPSHOT"
+fi
+
+info "Build complete, setting STACK_VERSION to $STACK_VERSION."
+
 info "Generated Artifacts"
 for file in build/logstash-*; do shasum $file;done
 

diff --git a/.buildkite/scripts/dra/generatesteps.py b/.buildkite/scripts/dra/generatesteps.py
@@ -3,6 +3,8 @@
 
 import yaml
 
+YAML_HEADER = '# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json\n'
+
 def to_bk_key_friendly_string(key):
     """
     Convert and return key to an acceptable format for Buildkite's key: field
@@ -106,6 +108,7 @@ def build_steps_to_yaml(branch, workflow_type):
 if __name__ == "__main__":
     try:
         workflow_type = os.environ["WORKFLOW_TYPE"]
+        version_qualifier = os.environ.get("VERSION_QUALIFIER", "")
     except ImportError:
         print(f"Missing env variable WORKFLOW_TYPE. Use export WORKFLOW_TYPE=<staging|snapshot>\n.Exiting.")
         exit(1)
@@ -114,18 +117,25 @@ def build_steps_to_yaml(branch, workflow_type):
 
     structure = {"steps": []}
 
-    # Group defining parallel steps that build and save artifacts
-    group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}")
-
-    structure["steps"].append({
-        "group": f":Build Artifacts - {workflow_type.upper()}",
-        "key": group_key,
-        "steps": build_steps_to_yaml(branch, workflow_type),
-    })
-
-    # Final step: pull artifacts built above and publish them via the release-manager
-    structure["steps"].extend(
-        yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)),
-    )
-
-    print('# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json\n' + yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False))
+    if workflow_type.upper() == "SNAPSHOT" and len(version_qualifier)>0:
+        structure["steps"].append({
+            "label": f"no-op pipeline because prerelease builds (VERSION_QUALIFIER is set to [{version_qualifier}]) don't support the [{workflow_type}] workflow",
+            "command": ":",
+            "skip": "VERSION_QUALIFIER (prerelease builds) not supported with SNAPSHOT DRA",
+        })
+    else:
+        # Group defining parallel steps that build and save artifacts
+        group_key = to_bk_key_friendly_string(f"logstash_dra_{workflow_type}")
+
+        structure["steps"].append({
+            "group": f":Build Artifacts - {workflow_type.upper()}",
+            "key": group_key,
+            "steps": build_steps_to_yaml(branch, workflow_type),
+        })
+
+        # Final step: pull artifacts built above and publish them via the release-manager
+        structure["steps"].extend(
+            yaml.safe_load(publish_dra_step(branch, workflow_type, depends_on=group_key)),
+        )
+
+    print(YAML_HEADER + yaml.dump(structure, Dumper=yaml.Dumper, sort_keys=False))
diff --git a/.buildkite/scripts/dra/publish.sh b/.buildkite/scripts/dra/publish.sh
@@ -7,38 +7,34 @@ echo "####################################################################"
 
 source ./$(dirname "$0")/common.sh
 
-PLAIN_STACK_VERSION=$STACK_VERSION
+# DRA_BRANCH can be used for manually testing packaging with PRs
+# e.g. define `DRA_BRANCH="main"` and `RUN_SNAPSHOT="true"` under Options/Environment Variables in the Buildkite UI after clicking new Build
+BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
 
 # This is the branch selector that needs to be passed to the release-manager
 # It has to be the name of the branch which originates the artifacts.
 RELEASE_VER=`cat versions.yml | sed -n 's/^logstash\:[[:space:]]\([[:digit:]]*\.[[:digit:]]*\)\.[[:digit:]]*$/\1/p'`
 if [ -n "$(git ls-remote --heads origin $RELEASE_VER)" ] ; then
     RELEASE_BRANCH=$RELEASE_VER
 else
-    RELEASE_BRANCH="${BUILDKITE_BRANCH:="main"}"
+    RELEASE_BRANCH="${BRANCH:="main"}"
 fi
 echo "RELEASE BRANCH: $RELEASE_BRANCH"
 
-if [ -n "$VERSION_QUALIFIER_OPT" ]; then
-  # Qualifier is passed from CI as optional field and specify the version postfix
-  # in case of alpha or beta releases:
-  # e.g: 8.0.0-alpha1
-  STACK_VERSION="${STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
-  PLAIN_STACK_VERSION="${PLAIN_STACK_VERSION}-${VERSION_QUALIFIER_OPT}"
-fi
+VERSION_QUALIFIER="${VERSION_QUALIFIER:=""}"
 
 case "$WORKFLOW_TYPE" in
     snapshot)
-        STACK_VERSION=${STACK_VERSION}-SNAPSHOT
+        :
         ;;
     staging)
         ;;
     *)
-        error "Worklflow (WORKFLOW_TYPE variable) is not set, exiting..."
+        error "Workflow (WORKFLOW_TYPE variable) is not set, exiting..."
         ;;
 esac
 
-info "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH}"
+info "Uploading artifacts for ${WORKFLOW_TYPE} workflow on branch: ${RELEASE_BRANCH} for version: ${STACK_VERSION} with version_qualifier: ${VERSION_QUALIFIER}"
 
 if [ "$RELEASE_VER" != "7.17" ]; then
   # Version 7.17.x doesn't generates ARM artifacts for Darwin
@@ -56,7 +52,16 @@ rm -f build/logstash-ubi8-${STACK_VERSION}-docker-image-aarch64.tar.gz
 info "Downloaded ARTIFACTS sha report"
 for file in build/logstash-*; do shasum $file;done
 
-mv build/distributions/dependencies-reports/logstash-${STACK_VERSION}.csv build/distributions/dependencies-${STACK_VERSION}.csv
+FINAL_VERSION=$STACK_VERSION
+if [[ -n "$VERSION_QUALIFIER" ]]; then
+  FINAL_VERSION="$FINAL_VERSION-${VERSION_QUALIFIER}"
+fi
+
+if [[ "$WORKFLOW_TYPE" == "snapshot" ]]; then
+    FINAL_VERSION="${STACK_VERSION}-SNAPSHOT"
+fi
+
+mv build/distributions/dependencies-reports/logstash-${FINAL_VERSION}.csv build/distributions/dependencies-${FINAL_VERSION}.csv
 
 # set required permissions on artifacts and directory
 chmod -R a+r build/*
@@ -74,6 +79,22 @@ release_manager_login
 # ensure the latest image has been pulled
 docker pull docker.elastic.co/infra/release-manager:latest
 
+echo "+++ :clipboard: Listing DRA artifacts for version [$STACK_VERSION], branch [$RELEASE_BRANCH], workflow [$WORKFLOW_TYPE], QUALIFIER [$VERSION_QUALIFIER]"
+docker run --rm \
+        --name release-manager \
+        -e VAULT_ROLE_ID \
+        -e VAULT_SECRET_ID \
+        --mount type=bind,readonly=false,src="$PWD",target=/artifacts \
+        docker.elastic.co/infra/release-manager:latest \
+          cli list \
+          --project logstash \
+          --branch "${RELEASE_BRANCH}" \
+          --commit "$(git rev-parse HEAD)" \
+          --workflow "${WORKFLOW_TYPE}" \
+          --version "${STACK_VERSION}" \
+          --artifact-set main \
+          --qualifier "${VERSION_QUALIFIER}"
+
 info "Running the release manager ..."
 
 # collect the artifacts for use with the unified build
@@ -89,8 +110,9 @@ docker run --rm \
       --branch ${RELEASE_BRANCH} \
       --commit "$(git rev-parse HEAD)" \
       --workflow "${WORKFLOW_TYPE}" \
-      --version "${PLAIN_STACK_VERSION}" \
+      --version "${STACK_VERSION}" \
       --artifact-set main \
+      --qualifier "${VERSION_QUALIFIER}" \
       ${DRA_DRY_RUN} | tee rm-output.txt
 
 # extract the summary URL from a release manager output line like: