ci: include build version in file name of build artifacts (scionproto…

…#4525)

Previously the artifacts had static, generic names. This has always felt
a bit clumsy, as downloading artifacts from multiple builds with a
browser would usually result in automatically deduplicated file names
like `scion(5).tar`.

After learning how the buildkite agent can interpolate variables in the
pipeline during the pipeline upload
(https://buildkite.com/docs/pipelines/environment-variables#variable-interpolation),
it seems easy to improve this.
Now the artifacts include the version, matching the name of the
artifacts in releases.

.buildkite/hooks/pre-command

@@ -16,8 +16,24 @@ set -euo pipefail
 echo "~~~ Increase receive network buffer size"
 sudo sysctl -w net.core.rmem_max=1048576
 
-echo "~~~ Setting up bazel environment"
 
+# Export SCION_VERSION as environment variable, unless it's set from outside (e.g. as variable for this build)
+# Note that this precommand hook runs even in "step zero" of the pipeline, where we `buildkite-agent upload'.
+# With this, the SCION_VERSION can be interpolated by the agent throughout the pipeline yaml.
+if [ -z ${SCION_VERSION+x} ]; then
+    echo "~~~ Export SCION_VERSION"
+    if [ "$BUILDKITE_PIPELINE_SLUG" == "scion" ]; then
+        # Shorten the git version to omit commit information, improving cache reuse.
+        # The format of git-version is "<tag>-<number-of-commits-since-the-tag>-<commit-short-hash>"
+        # This will be shortened to "<tag>-modified-ci"
+        export SCION_VERSION=$(tools/git-version | sed 's/-.*/-modified-ci/')
+    else
+        export SCION_VERSION=$(tools/git-version)
+    fi
+    echo SCION_VERSION=${SCION_VERSION}
+fi
+
+echo "~~~ Setting up bazel environment"
 if [ -z ${BAZEL_REMOTE_S3_ACCESS_KEY_ID+x} ]; then
     echo "S3 env not set, not starting bazel remote proxy"
     exit 0
@@ -30,12 +46,6 @@ rm -f $HOME/.bazelrc
 # --nostamp is required for better caching (only on non-release jobs).
 if [ "$BUILDKITE_PIPELINE_SLUG" == "scion" ]; then
     echo "build --nostamp" > $HOME/.bazelrc
-    if [ -z ${SCION_VERSION+x} ]; then
-        # Shorten the git version to omit commit information, improving cache reuse.
-        # The format of git-version is "<tag>-<number-of-commits-since-the-tag>-<commit-short-hash>"
-        # This will be shortened to "<tag>-modified-ci"
-        export SCION_VERSION=$(tools/git-version | sed 's/-.*/-modified-ci/')
-    fi
 else
     echo "build --stamp" > $HOME/.bazelrc
 fi

.buildkite/pipeline.yml

@@ -7,15 +7,18 @@ steps:
       - bazel build --verbose_failures //docker:prod //docker:test
     key: build
     artifact_paths:
-      - "bazel-bin/scion.tar"
-      - "bazel-bin/scion-ci.tar"
+      - "scion_${SCION_VERSION}_amd64_linux.tar.gz"     # Note: SCION_VERSION interpolated by buildkite agent uploading pipeline, see pre-command hook
+      - "scion-ci_${SCION_VERSION}_amd64_linux.tar.gz"
     plugins:
       - scionproto/metahook#v0.3.0:
+          pre-artifact: |
+            gzip --to-stdout bazel-bin/scion.tar > scion_${SCION_VERSION}_amd64_linux.tar.gz
+            gzip --to-stdout bazel-bin/scion-ci.tar > scion-ci_${SCION_VERSION}_amd64_linux.tar.gz
           post-artifact: |
             cat << EOF | buildkite-agent annotate --style "info" --context "binaries"
             #### Build outputs
-            - <a href="artifact://bazel-bin/scion.tar">SCION binaries</a>
-            - <a href="artifact://bazel-bin/scion-ci.tar">SCION test tools and utilities</a>
+            - <a href="artifact://scion_${SCION_VERSION}_amd64_linux.tar.gz">SCION binaries</a>
+            - <a href="artifact://scion-ci_${SCION_VERSION}_amd64_linux.tar.gz">SCION test tools and utilities</a>
             EOF
     retry: &automatic-retry
       automatic:
@@ -25,28 +28,31 @@ steps:
   - wait
   - label: "Package :debian: :openwrt:"
     command: |
-      version=${SCION_VERSION:-$(tools/git-version)}
-      make dist-deb BFLAGS="--file_name_version=$${version}"
-      make dist-openwrt BFLAGS="--file_name_version=$${version}"
-      cd installables;
-      tar -chaf scion-deb-amd64.tar.gz *_$${version}_amd64.deb
-      tar -chaf scion-deb-arm64.tar.gz *_$${version}_arm64.deb
-      tar -chaf scion-deb-i386.tar.gz *_$${version}_i386.deb
-      tar -chaf scion-deb-armel.tar.gz *_$${version}_armel.deb
-      tar -chaf scion-openwrt-x86_64.tar.gz *_$${version}_x86_64.ipk
+      make dist-deb BFLAGS="--file_name_version=${SCION_VERSION}"
+      make dist-openwrt BFLAGS="--file_name_version=${SCION_VERSION}"
     artifact_paths:
-      - "installables/scion-*.tar.gz"
+      - "installables/scion_*.tar.gz"
     plugins:
       - scionproto/metahook#v0.3.0:
+          pre-artifact: |
+            set -x
+            pushd installables
+            tar -chaf scion_${SCION_VERSION}_deb_amd64.tar.gz *_${SCION_VERSION}_amd64.deb
+            tar -chaf scion_${SCION_VERSION}_deb_arm64.tar.gz *_${SCION_VERSION}_arm64.deb
+            tar -chaf scion_${SCION_VERSION}_deb_i386.tar.gz *_${SCION_VERSION}_i386.deb
+            tar -chaf scion_${SCION_VERSION}_deb_armel.tar.gz *_${SCION_VERSION}_armel.deb
+            tar -chaf scion_${SCION_VERSION}_openwrt_x86_64.tar.gz *_${SCION_VERSION}_x86_64.ipk
+            popd
+            ls installables
           post-artifact: |
             cat << EOF | buildkite-agent annotate --style "info" --context "packages"
             #### Packages :debian:
-            - <a href="artifact://installables/scion-deb-amd64.tar.gz">amd64</a>
-            - <a href="artifact://installables/scion-deb-arm64.tar.gz">arm64</a>
-            - <a href="artifact://installables/scion-deb-i386.tar.gz">i386</a>
-            - <a href="artifact://installables/scion-deb-armel.tar.gz">armel</a>
+            - <a href="artifact://installables/scion_${SCION_VERSION}_deb_amd64.tar.gz">amd64</a>
+            - <a href="artifact://installables/scion_${SCION_VERSION}_deb_arm64.tar.gz">arm64</a>
+            - <a href="artifact://installables/scion_${SCION_VERSION}_deb_i386.tar.gz">i386</a>
+            - <a href="artifact://installables/scion_${SCION_VERSION}_deb_armel.tar.gz">armel</a>
             #### Packages :openwrt:
-            - <a href="artifact://installables/scion-openwrt-x86_64.tar.gz">x86_64</a>
+            - <a href="artifact://installables/scion_${SCION_VERSION}_openwrt_x86_64.tar.gz">x86_64</a>
             EOF
     key: dist
     retry: *automatic-retry