ci: upload binaries as artifacts (scionproto#4391)

Upload binaries built in the build step as build artifacts. Add a buildkite "annotation" to give quick access to these archive with these binaries. Inlcudes some cleanup around the uploading of artifacts and related hooks. Now only the handling of the bazel-remote and go-module-proxy is left in the pre-command/pre-exit hooks. The rest is moved into step-specific metahook-hooks. This includes collecting the test outputs artifacts -- this is kept as archives instead of individual files, just because the number of individual test output files seems somewhat overwhelming in the buildkite UI.
matzf · Sep 19, 2023 · 4686186 · 4686186
1 parent 010baef
commit 4686186
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 117 deletions.
diff --git a/.buildkite/cleanup-leftovers.sh b/.buildkite/cleanup-leftovers.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo "~~~ Cleaning up any leftovers"
+cntrs="$(docker ps -aq | grep -v -f <(docker ps -q --filter "name=go-module-proxy" --filter "name=bazel-remote-cache"))"
+[ -n "$cntrs" ] && { echo "Remove leftover containers..."; docker rm -f $cntrs; }
+echo "Remove leftover networks"
+docker network prune -f
+echo "Remove leftover volumes"
+docker volume prune -f
+
+rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts test-out.tar.gz
diff --git a/.buildkite/hooks/pre-artifact b/.buildkite/hooks/pre-artifact
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -11,15 +11,12 @@ printenv PATH
 # Install build tools on first job for this runner.
 .buildkite/provision-agent.sh
 
-# Clean up left-overs from previous run
-PRE_COMMAND_SETUP=true . .buildkite/hooks/pre-exit
-
 set -euo pipefail
 
-echo "--- Increase receive network buffer size"
+echo "~~~ Increase receive network buffer size"
 sudo sysctl -w net.core.rmem_max=1048576
 
-echo "--- Setting up bazel environment"
+echo "~~~ Setting up bazel environment"
 
 if [ -z ${BAZEL_REMOTE_S3_ACCESS_KEY_ID+x} ]; then
     echo "S3 env not set, not starting bazel remote proxy"
@@ -41,12 +38,12 @@ else
 fi
 echo "test --test_env CI" >> $HOME/.bazelrc
 
-echo "--- Starting bazel remote cache proxy"
+echo "~~~ Starting bazel remote cache proxy"
 
 # Start bazel remote cache proxy for S3
 # Note that S3 keys are injected by buildkite, see
 # https://buildkite.com/docs/pipelines/secrets#storing-secrets-with-the-elastic-ci-stack-for-aws
 docker-compose -f .buildkite/hooks/bazel-remote.yml -p bazel_remote up -d
 
-echo "--- Starting go module proxy"
+echo "~~~ Starting go module proxy"
 docker-compose -f .buildkite/hooks/go-module-proxy.yml -p athens up -d
diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit
@@ -1,8 +1,7 @@
 #!/bin/bash
 
-
-if [ -f ".buildkite/hooks/bazel-remote.yml" -a -z "$PRE_COMMAND_SETUP" ]; then
-    echo "--- Uploading bazel-remote and go-module-proxy logs/metrics"
+if [ -f ".buildkite/hooks/bazel-remote.yml" ]; then
+    echo "~~~ Uploading bazel-remote and go-module-proxy logs/metrics"
     curl http://localhost:8080/metrics > bazel-remote-cache.metrics
     docker logs bazel-remote-cache &> bazel-remote-cache.log
 
@@ -11,19 +10,3 @@ if [ -f ".buildkite/hooks/bazel-remote.yml" -a -z "$PRE_COMMAND_SETUP" ]; then
 
     buildkite-agent artifact upload "bazel-remote-cache.*;go-module-proxy.*"
 fi
-
-echo "--- Cleaning up the topology"
-
-./scion.sh topo_clean
-
-echo "--- Cleaning up docker containers/networks/volumes"
-cntrs="$(docker ps -aq | grep -v -f <(docker ps -q --filter "name=go-module-proxy" --filter "name=bazel-remote-cache"))"
-[ -n "$cntrs" ] && { echo "Remove leftover containers..."; docker rm -f $cntrs; }
-
-echo "Remove leftover networks"
-docker network prune -f
-echo "Remove leftover volumes"
-docker volume prune -f
-
-echo "--- Cleaning up logs and artifacts"
-rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -3,9 +3,20 @@ env:
 steps:
   - label: "Build :bazel:"
     command:
-      - bazel build --verbose_failures --announce_rc //:all
+      - bazel build --verbose_failures --announce_rc //:scion //:scion-ci
       - bazel run --verbose_failures //docker:prod //docker:test
     key: build
+    artifact_paths:
+      - "bazel-bin/scion.tar"
+      - "bazel-bin/scion-ci.tar"
+    plugins:
+      - scionproto/metahook#v0.3.0:
+          post-artifact: |
+            cat << EOF | buildkite-agent annotate --style "info"
+            #### Build outputs
+            - <a href="artifact://bazel-bin/scion.tar">SCION binaries</a>
+            - <a href="artifact://bazel-bin/scion-ci.tar">SCION test tools and utilities</a>
+            EOF
     retry: &automatic-retry
       automatic:
         - exit_status: -1 # Agent was lost
@@ -16,8 +27,11 @@ steps:
     command:
       - bazel test --config=race --config=unit_all
     key: unit_tests
+    plugins:
+      - scionproto/metahook#v0.3.0:
+          pre-artifact: tar -chaf bazel-testlogs.tar.gz bazel-testlogs
     artifact_paths:
-      - "artifacts.out/**/*"
+      - bazel-testlogs.tar.gz
     retry: *automatic-retry
     timeout_in_minutes: 20
   - label: "Lint :bash:"
@@ -69,14 +83,20 @@ steps:
         - tools/await-connectivity
         - ./bin/scion_integration || ( echo "^^^ +++" && false )
         - ./bin/end2end_integration || ( echo "^^^ +++" && false )
-      plugins: &shutdown-scion-post-command
+      plugins: &scion-run-hooks
         - scionproto/metahook#v0.3.0:
+            pre-command: .buildkite/cleanup-leftovers.sh
             post-command: |
-              echo "--- Shutting down SCION topology"
+              echo "~~~ Shutting down SCION topology"
               ./scion.sh stop
-              echo "SCION topology successfully shut down"
-      artifact_paths:
-        - "artifacts.out/**/*"
+            pre-artifact: |
+              if [ -f "gen/scion-dc.yml" ]; then
+                  tools/dc collect_logs scion logs/
+              fi
+              tar -chaf test-out.tar.gz $(ls -d logs traces gen gen-cache) # ls -d to filter missing directories
+            pre-exit: .buildkite/cleanup-leftovers.sh
+      artifact_paths: &scion-run-artifact-paths
+        - test-out.tar.gz
       timeout_in_minutes: 15
       key: e2e_integration_tests_v2
       retry: *automatic-retry
@@ -90,9 +110,8 @@ steps:
         - tools/await-connectivity
         - ./bin/end2end_integration || ( echo "^^^ +++" && false )
         - ./tools/integration/revocation_test.sh
-      plugins: *shutdown-scion-post-command
-      artifact_paths:
-        - "artifacts.out/**/*"
+      plugins: *scion-run-hooks
+      artifact_paths: *scion-run-artifact-paths
       timeout_in_minutes: 15
       key: e2e_revocation_test_v2
       retry: *automatic-retry
@@ -106,9 +125,8 @@ steps:
         - tools/await-connectivity
         - echo "--- run tests"
         - ./bin/end2end_integration -d || ( echo "^^^ +++" && false )
-      plugins: *shutdown-scion-post-command
-      artifact_paths:
-        - "artifacts.out/**/*"
+      plugins: *scion-run-hooks
+      artifact_paths: *scion-run-artifact-paths
       timeout_in_minutes: 15
       key: docker_integration_e2e_default
       retry: *automatic-retry
diff --git a/.buildkite/pipeline_lib.sh b/.buildkite/pipeline_lib.sh
@@ -44,8 +44,13 @@ gen_bazel_test_steps() {
         echo "        command:"
         echo "          - bazel test --test_output=streamed $test $args $cache"
         echo "        key: \"${name////_}\""
+        echo "        plugins:"
+        echo "          - scionproto/metahook#v0.3.0:"
+        echo "              pre-command: .buildkite/cleanup-leftovers.sh"
+        echo "              pre-artifact: tar -chaf bazel-testlogs.tar.gz bazel-testlogs"
+        echo "              pre-exit: .buildkite/cleanup-leftovers.sh"
         echo "        artifact_paths:"
-        echo "          - \"artifacts.out/**/*\""
+        echo "          - \"bazel-testlogs.tar.gz\""
         echo "        timeout_in_minutes: 20"
         echo "        retry:"
         echo "          automatic:"

diff --git a/README.md b/README.md
@@ -3,7 +3,7 @@
 [![Slack chat](https://img.shields.io/badge/chat%20on-slack-blue?logo=slack)](https://scionproto.slack.com)
 [![ReadTheDocs](https://img.shields.io/badge/doc-reference-blue?version=latest&style=flat&label=docs&logo=read-the-docs&logoColor=white)](https://docs.scion.org/en/latest)
 [![Documentation](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white)](https://pkg.go.dev/github.com/scionproto/scion)
-[![Build Status](https://badge.buildkite.com/e7ca347d947c23883ad7c3a4d091c2df5ae7feb52b238d29a1.svg?branch=master)](https://buildkite.com/scionproto/scion)
+[![Nightly Build](https://badge.buildkite.com/b70b65b38a75eb8724f41a6f1203c9327cfb767f07a0c1934e.svg)](https://buildkite.com/scionproto/scion-nightly/builds/latest)
 [![Go Report Card](https://goreportcard.com/badge/github.com/scionproto/scion)](https://goreportcard.com/report/github.com/scionproto/scion)
 [![GitHub issues](https://img.shields.io/github/issues/scionproto/scion/help%20wanted.svg?label=help%20wanted&color=purple)](https://github.com/scionproto/scion/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22)
 [![GitHub issues](https://img.shields.io/github/issues/scionproto/scion/good%20first%20issue.svg?label=good%20first%20issue&color=purple)](https://github.com/scionproto/scion/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
@@ -31,6 +31,8 @@ To find out how to work with SCION, please visit our [documentation
 site](https://docs.scion.org/en/latest/dev/setup.html)
 for instructions on how to install build dependencies, build and run SCION.
 
+Pre-built binaries for x86-64 Linux are available from the [latest nightly build](https://buildkite.com/scionproto/scion-nightly/builds/latest).
+
 ## Contributing
 
 Interested in contribution to the SCION project? Please visit our