From 3f5e87c8cabc19f8aae346e5de63b7d21c164bf1 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Tue, 3 Sep 2024 16:56:44 -0300 Subject: [PATCH] Normalize build-binaries targets (#5924) ## Which problem is this PR solving? - Part of #5889 - `build-binaries-***` Makefile targets sometimes include os-arch, sometimes just os, sometimes just arch - it's a mess ## Description of the changes - normalize all targets to be `build-binaries-$os-$arch` - change `ci-build-binaries.yml` workflow to use `make echo-platforms` for the jobs matrix ## How was this change tested? - ci --------- Signed-off-by: Yuri Shkuro --- .github/workflows/ci-build-binaries.yml | 41 ++++++++----------- Makefile.BuildBinaries.mk | 52 ++++++++++++------------- Makefile.Crossdock.mk | 6 +-- scripts/build-all-in-one-image.sh | 1 - scripts/build-upload-docker-images.sh | 3 +- scripts/cassandra-integration-test.sh | 1 - scripts/es-integration-test.sh | 1 - scripts/kafka-integration-test.sh | 2 - scripts/package-deploy.sh | 2 +- scripts/platforms-to-gh-matrix.sh | 21 ++++++++++ 10 files changed, 68 insertions(+), 62 deletions(-) create mode 100755 scripts/platforms-to-gh-matrix.sh diff --git a/.github/workflows/ci-build-binaries.yml b/.github/workflows/ci-build-binaries.yml index 7e4764eb012..6c6a2cb8ef5 100644 --- a/.github/workflows/ci-build-binaries.yml +++ b/.github/workflows/ci-build-binaries.yml @@ -16,29 +16,25 @@ permissions: contents: read jobs: + generate-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: define matrix + id: set-matrix + run: | + echo "matrix=$(bash scripts/platforms-to-gh-matrix.sh)" >> $GITHUB_OUTPUT + build-binaries: + needs: generate-matrix runs-on: ubuntu-latest strategy: - matrix: - platform: - - name: linux - task: build-binaries-linux - - name: windows - task: build-binaries-windows - - name: osx - task: build-binaries-darwin - - name: osx-m1 - task: build-binaries-darwin-arm64 - - name: system/390 - task: build-binaries-s390x - - name: arm - task: build-binaries-arm64 - - name: powerpc - task: build-binaries-ppc64le - name: build binaries for ${{ matrix.platform.name }} + matrix: ${{fromJson(needs.generate-matrix.outputs.matrix)}} + name: build-binaries-${{ matrix.os }}-${{ matrix.arch }} steps: - - name: Harden Runner - uses: step-security/harden-runner@0d381219ddf674d61a7572ddd19d7941e271515c # v2.9.0 + - uses: step-security/harden-runner@0d381219ddf674d61a7572ddd19d7941e271515c # v2.9.0 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs @@ -57,11 +53,8 @@ jobs: - name: Setup Node.js version uses: ./.github/actions/setup-node.js - - name: Export BRANCH variable - uses: ./.github/actions/setup-branch - - name: Install tools run: make install-ci - - name: Build binaries - run: make ${{ matrix.platform.task }} + - name: Build platform binaries + run: make build-binaries-${{ matrix.os }}-${{ matrix.arch }} diff --git a/Makefile.BuildBinaries.mk b/Makefile.BuildBinaries.mk index bade512f140..37ec1f0083e 100644 --- a/Makefile.BuildBinaries.mk +++ b/Makefile.BuildBinaries.mk @@ -1,7 +1,9 @@ # Copyright (c) 2023 The Jaeger Authors. # SPDX-License-Identifier: Apache-2.0 -GOBUILD=CGO_ENABLED=0 installsuffix=cgo $(GO) build -trimpath +# This command expects $GOOS/$GOARCH env variables set to reflect the desired target platform. +GOBUILD=echo "building for $$(go env GOOS)-$$(go env GOARCH)"; \ + CGO_ENABLED=0 installsuffix=cgo $(GO) build -trimpath ifeq ($(DEBUG_BINARY),) DISABLE_OPTIMIZATIONS = @@ -29,13 +31,11 @@ jaeger-ui/packages/jaeger-ui/build/index.html: .PHONY: rebuild-ui rebuild-ui: + @echo "::group::rebuild-ui logs" bash ./scripts/rebuild-ui.sh @echo "NOTE: This target only rebuilds the UI assets inside jaeger-ui/packages/jaeger-ui/build/." @echo "NOTE: To make them usable from query-service run 'make build-ui'." - -.PHONY: build-all-in-one-linux -build-all-in-one-linux: - GOOS=linux $(MAKE) build-all-in-one + @echo "::endgroup::" .PHONY: build-examples build-examples: @@ -101,38 +101,35 @@ build-ingester: _build-a-binary-ingester$(SUFFIX)-$(GOOS)-$(GOARCH) build-remote-storage: BIN_NAME = remote-storage build-remote-storage: _build-a-binary-remote-storage$(SUFFIX)-$(GOOS)-$(GOARCH) -.PHONY: build-binaries-linux -build-binaries-linux: build-binaries-amd64 - -.PHONY: build-binaries-amd64 -build-binaries-amd64: +.PHONY: build-binaries-linux-amd64 +build-binaries-linux-amd64: GOOS=linux GOARCH=amd64 $(MAKE) _build-platform-binaries -# helper targets defined in Makefile.Windows.mk -.PHONY: build-binaries-windows -build-binaries-windows: +# helper sysp targets are defined in Makefile.Windows.mk +.PHONY: build-binaries-windows-amd64 +build-binaries-windows-amd64: $(MAKE) _build-syso GOOS=windows GOARCH=amd64 $(MAKE) _build-platform-binaries $(MAKE) _clean-syso -.PHONY: build-binaries-darwin -build-binaries-darwin: +.PHONY: build-binaries-darwin-amd64 +build-binaries-darwin-amd64: GOOS=darwin GOARCH=amd64 $(MAKE) _build-platform-binaries .PHONY: build-binaries-darwin-arm64 build-binaries-darwin-arm64: GOOS=darwin GOARCH=arm64 $(MAKE) _build-platform-binaries -.PHONY: build-binaries-s390x -build-binaries-s390x: +.PHONY: build-binaries-linux-s390x +build-binaries-linux-s390x: GOOS=linux GOARCH=s390x $(MAKE) _build-platform-binaries -.PHONY: build-binaries-arm64 -build-binaries-arm64: +.PHONY: build-binaries-linux-arm64 +build-binaries-linux-arm64: GOOS=linux GOARCH=arm64 $(MAKE) _build-platform-binaries -.PHONY: build-binaries-ppc64le -build-binaries-ppc64le: +.PHONY: build-binaries-linux-ppc64le +build-binaries-linux-ppc64le: GOOS=linux GOARCH=ppc64le $(MAKE) _build-platform-binaries # build all binaries for one specific platform GOOS/GOARCH @@ -156,6 +153,7 @@ _build-platform-binaries: \ # build binaries that support DEBUG release, for one specific platform GOOS/GOARCH .PHONY: _build-platform-binaries-debug +_build-platform-binaries-debug: _build-platform-binaries-debug: \ build-agent \ build-collector \ @@ -167,10 +165,10 @@ _build-platform-binaries-debug: \ .PHONY: build-all-platforms build-all-platforms: \ - build-binaries-linux \ - build-binaries-windows \ - build-binaries-darwin \ + build-binaries-linux-amd64 \ + build-binaries-windows-amd64 \ + build-binaries-darwin-amd64 \ build-binaries-darwin-arm64 \ - build-binaries-s390x \ - build-binaries-arm64 \ - build-binaries-ppc64le + build-binaries-linux-s390x \ + build-binaries-linux-arm64 \ + build-binaries-linux-ppc64le diff --git a/Makefile.Crossdock.mk b/Makefile.Crossdock.mk index 56d6f595767..14a0a1f3ed9 100644 --- a/Makefile.Crossdock.mk +++ b/Makefile.Crossdock.mk @@ -19,7 +19,7 @@ build-crossdock-ui-placeholder: $(MAKE) build-ui .PHONY: build-crossdock -build-crossdock: build-crossdock-ui-placeholder build-binaries-linux build-crossdock-linux docker-images-cassandra crossdock-docker-images-jaeger-backend +build-crossdock: build-crossdock-ui-placeholder build-binaries-linux-$(GOARCH) build-crossdock-binary docker-images-cassandra crossdock-docker-images-jaeger-backend docker build -t $(DOCKER_NAMESPACE)/test-driver:${DOCKER_TAG} --build-arg TARGETARCH=$(GOARCH) crossdock/ @echo "Finished building test-driver ==============" ; \ @@ -28,11 +28,11 @@ build-and-run-crossdock: build-crossdock make crossdock .PHONY: build-crossdock-fresh -build-crossdock-fresh: build-crossdock-linux +build-crossdock-fresh: build-crossdock-binary make crossdock-fresh .PHONY: crossdock-docker-images-jaeger-backend -crossdock-docker-images-jaeger-backend: PLATFORMS=linux/$(shell go env GOARCH) +crossdock-docker-images-jaeger-backend: PLATFORMS=linux/$(GOARCH) crossdock-docker-images-jaeger-backend: create-baseimg create-fake-debugimg for component in "jaeger-agent" "jaeger-collector" "jaeger-query" "jaeger-ingester" "all-in-one" ; do \ regex="jaeger-(.*)"; \ diff --git a/scripts/build-all-in-one-image.sh b/scripts/build-all-in-one-image.sh index c26757ddecd..5bec81869bb 100755 --- a/scripts/build-all-in-one-image.sh +++ b/scripts/build-all-in-one-image.sh @@ -81,7 +81,6 @@ run_integration_test() { # Loop through each platform (separated by commas) for platform in $(echo "$platforms" | tr ',' ' '); do - # Extract the architecture from the platform string arch=${platform##*/} # Remove everything before the last slash make "build-${BINARY}" GOOS=linux GOARCH="${arch}" done diff --git a/scripts/build-upload-docker-images.sh b/scripts/build-upload-docker-images.sh index 6775c85f217..1d1cecfe3ef 100755 --- a/scripts/build-upload-docker-images.sh +++ b/scripts/build-upload-docker-images.sh @@ -40,9 +40,8 @@ set -x # Loop through each platform (separated by commas) for platform in $(echo "$platforms" | tr ',' ' '); do - # Extract the architecture from the platform string arch=${platform##*/} # Remove everything before the last slash - make "build-binaries-$arch" + make "build-binaries-linux-${arch}" done if [[ "${add_debugger}" == "N" ]]; then diff --git a/scripts/cassandra-integration-test.sh b/scripts/cassandra-integration-test.sh index 849c4109981..53a547ddb45 100755 --- a/scripts/cassandra-integration-test.sh +++ b/scripts/cassandra-integration-test.sh @@ -24,7 +24,6 @@ check_arg() { setup_cassandra() { local compose_file=$1 docker compose -f "$compose_file" up -d - echo "docker_compose_file=${compose_file}" >> "${GITHUB_OUTPUT:-/dev/null}" } dump_logs() { diff --git a/scripts/es-integration-test.sh b/scripts/es-integration-test.sh index d1e58af5e7d..a6c8b8296b6 100755 --- a/scripts/es-integration-test.sh +++ b/scripts/es-integration-test.sh @@ -29,7 +29,6 @@ check_arg() { setup_db() { local compose_file=$1 docker compose -f "${compose_file}" up -d - echo "docker_compose_file=${compose_file}" >> "${GITHUB_OUTPUT:-/dev/null}" } # check if the storage is up and running diff --git a/scripts/kafka-integration-test.sh b/scripts/kafka-integration-test.sh index e4a1d375b01..0777e7efb1d 100755 --- a/scripts/kafka-integration-test.sh +++ b/scripts/kafka-integration-test.sh @@ -6,8 +6,6 @@ set -euf -o pipefail compose_file="docker-compose/kafka-integration-test/docker-compose.yml" -echo "docker_compose_file=${compose_file}" >> "${GITHUB_OUTPUT:-/dev/null}" - jaeger_version="" manage_kafka="true" success="false" diff --git a/scripts/package-deploy.sh b/scripts/package-deploy.sh index 7453336b9fb..8f87ac2756e 100755 --- a/scripts/package-deploy.sh +++ b/scripts/package-deploy.sh @@ -115,7 +115,7 @@ mkdir deploy # Loop through each platform (separated by commas) for platform in $(echo "$platforms" | tr ',' ' '); do - os="${platform%%/*}" # Remove everything after the slash + os=${platform%%/*} # Remove everything after the slash arch=${platform##*/} # Remove everything before the last slash if [[ "$os" == "windows" ]]; then package tar "${os}-${arch}" .exe diff --git a/scripts/platforms-to-gh-matrix.sh b/scripts/platforms-to-gh-matrix.sh new file mode 100755 index 00000000000..55ff3127fe5 --- /dev/null +++ b/scripts/platforms-to-gh-matrix.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# +# Copyright (c) 2024 The Jaeger Authors. +# SPDX-License-Identifier: Apache-2.0 + +set -euf -o pipefail + +echo -n '{ "include": [ ' +first="true" +for pair in $(make echo-platforms | tr ',' ' '); do + os=$(echo "$pair" | cut -d '/' -f 1) + arch=$(echo "$pair" | cut -d '/' -f 2) + if [[ "$first" == "true" ]]; then + first="false" + else + echo -n ' ,' + fi + echo -n "{ \"os\": \"$os\", \"arch\": \"$arch\" }" +done + +echo "]}"