Wait for Buildbarn to become available in docker compose test

buildbarn · Mar 5, 2024 · b410ba0 · b410ba0
1 parent a8234ed
commit b410ba0
Show file tree

Hide file tree

Showing 7 changed files with 35 additions and 1 deletion.
diff --git a/.github/workflows/master.yaml b/.github/workflows/master.yaml
@@ -15,6 +15,10 @@
                "name": "Installing Bazelisk",
                "run": "bazelisk_fingerprint=231ec5ca8115e94c75a1f4fbada1a062b48822ca04f21f26e4cb1cd8973cd458 &&\n(echo \"${bazelisk_fingerprint} ${HOME}/.cache/bazelisk/bazel\" | sha256sum --check --quiet) || (\n  mkdir -p ~/.cache/bazelisk &&\n  curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 > ~/.cache/bazelisk/bazelisk.tmp &&\n  chmod +x ~/.cache/bazelisk/bazelisk.tmp &&\n  mv ~/.cache/bazelisk/bazelisk.tmp ~/.cache/bazelisk/bazel\n) &&\n(echo \"${bazelisk_fingerprint} ${HOME}/.cache/bazelisk/bazel\" | sha256sum --check --quiet) &&\necho \"~/.cache/bazelisk\" >> ${GITHUB_PATH}\n"
             },
+            {
+               "name": "Installing grpcurl",
+               "run": "mkdir -p ~/.cache/grpcurl &&\ncurl -L https://github.com/fullstorydev/grpcurl/releases/download/v1.8.9/grpcurl_1.8.9_linux_x86_64.tar.gz | tar -xz -C ~/.cache/grpcurl &&\necho \"~/.cache/grpcurl\" >> ${GITHUB_PATH}\n"
+            },
             {
                "name": "Install k3d",
                "run": "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash"

diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml
@@ -15,6 +15,10 @@
                "name": "Installing Bazelisk",
                "run": "bazelisk_fingerprint=231ec5ca8115e94c75a1f4fbada1a062b48822ca04f21f26e4cb1cd8973cd458 &&\n(echo \"${bazelisk_fingerprint} ${HOME}/.cache/bazelisk/bazel\" | sha256sum --check --quiet) || (\n  mkdir -p ~/.cache/bazelisk &&\n  curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 > ~/.cache/bazelisk/bazelisk.tmp &&\n  chmod +x ~/.cache/bazelisk/bazelisk.tmp &&\n  mv ~/.cache/bazelisk/bazelisk.tmp ~/.cache/bazelisk/bazel\n) &&\n(echo \"${bazelisk_fingerprint} ${HOME}/.cache/bazelisk/bazel\" | sha256sum --check --quiet) &&\necho \"~/.cache/bazelisk\" >> ${GITHUB_PATH}\n"
             },
+            {
+               "name": "Installing grpcurl",
+               "run": "mkdir -p ~/.cache/grpcurl &&\ncurl -L https://github.com/fullstorydev/grpcurl/releases/download/v1.8.9/grpcurl_1.8.9_linux_x86_64.tar.gz | tar -xz -C ~/.cache/grpcurl &&\necho \"~/.cache/grpcurl\" >> ${GITHUB_PATH}\n"
+            },
             {
                "name": "Install k3d",
                "run": "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash"

diff --git a/docker-compose/config/scheduler.jsonnet b/docker-compose/config/scheduler.jsonnet
@@ -13,6 +13,10 @@ local common = import 'common.libsonnet';
     listenAddresses: [':8983'],
     authenticationPolicy: { allow: {} },
   }],
+  buildQueueStateGrpcServers: [{
+    listenAddresses: [':8984'],
+    authenticationPolicy: { allow: {} },
+  }],
   browserUrl: common.browserUrl,
   contentAddressableStorage: common.blobstore.contentAddressableStorage,
   maximumMessageSizeBytes: common.maximumMessageSizeBytes,

diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml
@@ -42,10 +42,12 @@ services:
     expose:
     - 8982
     - 8983
+    - 8984
     - 9980
     ports:
     - 7982:7982
     - 8983:8983
+    - 8984:8984
     volumes:
     - ./config:/config
 

diff --git a/kubernetes/config/scheduler.jsonnet b/kubernetes/config/scheduler.jsonnet
@@ -13,6 +13,10 @@ local common = import 'common.libsonnet';
     listenAddresses: [':8983'],
     authenticationPolicy: { allow: {} },
   }],
+  buildQueueStateGrpcServers: [{
+    listenAddresses: [':8984'],
+    authenticationPolicy: { allow: {} },
+  }],
   browserUrl: common.browserUrl,
   contentAddressableStorage: common.blobstore.contentAddressableStorage,
   maximumMessageSizeBytes: common.maximumMessageSizeBytes,

diff --git a/tools/github_workflows/workflows_template.libsonnet b/tools/github_workflows/workflows_template.libsonnet
@@ -75,6 +75,14 @@
             echo "~/.cache/bazelisk" >> ${GITHUB_PATH}
           |||,
         },
+        {
+          name: 'Installing grpcurl',
+          run: |||
+            mkdir -p ~/.cache/grpcurl &&
+            curl -L https://github.com/fullstorydev/grpcurl/releases/download/v1.8.9/grpcurl_1.8.9_linux_x86_64.tar.gz | tar -xz -C ~/.cache/grpcurl &&
+            echo "~/.cache/grpcurl" >> ${GITHUB_PATH}
+          |||,
+        },
         {
           name: 'Install k3d',
           run: 'curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash',

diff --git a/tools/test-deployment-docker-compose.sh b/tools/test-deployment-docker-compose.sh
@@ -19,7 +19,15 @@ trap cleanup EXIT
 # --- Run remote execution ---
 rm -rf volumes/storage-*
 ./run.sh -d
-docker-compose up --wait || true
+# Wait for queues for all worker instance types to be available.
+docker compose up --wait frontend scheduler
+while : ; do
+    instance_name_prefixes=$(grpcurl --plaintext localhost:8984 buildbarn.buildqueuestate.BuildQueueState.ListPlatformQueues |
+        jq -r '.platformQueues | map(.name.instanceNamePrefix) | sort | join(",")')
+    [ "$instance_name_prefixes" != "fuse,hardlinking" ] || break
+    sleep 1
+done
+
 bazel_command_log="$(bazel info output_base)/command.log"
 bazel clean
 bazel test --color=no --curses=no --config=remote-ubuntu-22-04 --disk_cache= @abseil-hello//:hello_test