From a5b2dbea8c1d8783cdf7939d97363c98f6a59842 Mon Sep 17 00:00:00 2001 From: Fred Heinecke Date: Thu, 28 Mar 2024 16:20:23 -0500 Subject: [PATCH] Updated earthfile to fix bugs found when developing gha-exporter --- tools/ami-cleanup/Earthfile | 230 +++++++++++++++++++++++++----------- 1 file changed, 163 insertions(+), 67 deletions(-) diff --git a/tools/ami-cleanup/Earthfile b/tools/ami-cleanup/Earthfile index 859a669f..b09b50aa 100644 --- a/tools/ami-cleanup/Earthfile +++ b/tools/ami-cleanup/Earthfile @@ -1,21 +1,22 @@ -# These args will not be needed upon 0.8 release, which is scheduled for late January/ -# early February 2024. -VERSION --arg-scope-and-set --git-refs --global-cache 0.7 +VERSION 0.8 +# These args are unlikely to change between runs. If they do change then generally all targets should be reran +# without caching anyway. ARG --global GIT_TAG -ARG --global BINARY_NAME="ami-cleanup" -ARG --global IMAGE_NAME="ami-cleanup" +ARG --global PROJECT_NAME="ami-cleanup" +ARG --global BINARY_NAME="$PROJECT_NAME" +ARG --global IMAGE_NAME="$PROJECT_NAME" +ARG --global REPO_NAME="gravitational/shared-workflows" +ARG --global USEROS +ARG --global USERARCH # This target is used to setup a common Go environment used for both builds and tests. go-environment: - # Environment variables - ARG --required TARGETOS - ARG --required TARGETARCH - # Native arch is required because otherewise images will default to TARGETARCH, + # Native arch is required because otherwise images will default to TARGETARCH, # which is overridden by `--platform`. ARG --required NATIVEARCH - # This keeps the Go version set in a single place + # This keeps the Go version set in a single place. # A container is used to pin the `sed` dependency. `LOCALLY` could be used instead, but is # disallowed by the `--strict` Earthly flag which is used to help enfore reproducability. FROM --platform="linux/$NATIVEARCH" alpine:3.19.0 @@ -25,109 +26,106 @@ go-environment: # Run on the native architecture, but setup for cross compilation. FROM --platform="linux/$NATIVEARCH" "golang:$GO_VERSION" - ENV GOOS=$TARGETOS - ENV GOARCH=$TARGETARCH + # Ensure built binaries are statically linked + ENV CGO_ENABLED=0 WORKDIR /go/src - CACHE --sharing shared $(go env GOMODCACHE) - # Load the source and download modules - COPY src/ . + # Load the module file and download the modules + COPY src/go.mod . RUN go mod download -x +EXTRACT_VERSION: + FUNCTION + ARG --required GIT_TAG + ENV GIT_VERSION=$(echo "$GIT_TAG" | sed 's/.*-v/v/') + # Produces a single executable binary file for the target platform. -# This should generally be called as `earthly --platform= +binary`. +# This should generally be called as `earthly --GOOS= --GOARCH= +binary`. binary: - ARG --required TARGETPLATFORM - FROM --platform "$TARGETPLATFORM" +go-environment - # Caches are specific to a given target, so the GOCACHE is declared here as it - # is updated when builds run - CACHE --sharing shared --id gocache $(go env GOCACHE) + # These are automatically mapped to enviroment variables + ARG GOOS=$USEROS + ARG GOARCH=$USERARCH + + FROM +go-environment # Setup for the build + # `IF` statements essentially run as shell `if` statements, so a build context must be declared + # for them. LET LINKER_FLAGS="-s -w" IF [ -n "$GIT_TAG" ] + DO +EXTRACT_VERSION --GIT_TAG=$GIT_TAG ARG EARTHLY_GIT_SHORT_HASH - SET LINKER_FLAGS="$LINKER_FLAGS -X 'main.Version=$GIT_TAG+$EARTHLY_GIT_SHORT_HASH'" + SET LINKER_FLAGS="$LINKER_FLAGS -X 'main.Version=$GIT_VERSION+$EARTHLY_GIT_SHORT_HASH'" END LET BINARY_OUTPUT_PATH="../$BINARY_NAME" + # Caches are specific to a given target, so the GOCACHE is declared here rather than + # in the `go-environment` target. This ensures that the Go cache from a previous + # `go build` is reused, minimizing the work that needs to be done for Go code changes. + CACHE --sharing shared --id gocache $(go env GOCACHE) + # Do the actual build + COPY src/ . RUN go build -o "$BINARY_OUTPUT_PATH" -ldflags="$LINKER_FLAGS" cmd/main.go # Process the outputs - SAVE ARTIFACT "$BINARY_OUTPUT_PATH" AS LOCAL "outputs/$TARGETPLATFORM/$BINARY_NAME" - -# Same as `binary`, except the platform defaults to the local host. -local-binary: - # This is a workaround for the default TARGETOS value being the buildkit OS (linux), - # which is wrong when running on MacOS. Unfortunately this is not fixed by specifying - # TARGETOS under LOCALLY, because then it cannot be overridden via the platform arg. - ARG --required USERPLATFORM - BUILD --platform="$USERPLATFORM" +binary + SAVE ARTIFACT "$BINARY_OUTPUT_PATH" AS LOCAL "outputs/$GOOS/$GOARCH/$BINARY_NAME" # Produces a container image and multiarch manifest. These are automatically loaded into the # local Docker image cache. If multiple platforms are specified, then they are all added # under the same image. container-image: # Build args - ARG --required TARGETARCH - ARG --required NATIVEARCH - ARG CONTAINER_REGISTRY="" + ARG GOARCH=$USERARCH + ARG NATIVEARCH + ARG OCI_REGISTRY # Setup for build - # `IF` statements essentially run as shell `if` statements, so a build context must be declared - # for them. FROM --platform="linux/$NATIVEARCH" alpine:3.19.0 LET IMAGE_TAG="latest" IF [ -n "$GIT_TAG" ] - SET IMAGE_TAG="$GIT_TAG" + DO +EXTRACT_VERSION --GIT_TAG=$GIT_TAG + SET IMAGE_TAG="$GIT_VERSION" END # Do the actual build - RUN echo "FULL IMAGE NAME: $CONTAINER_REGISTRY$IMAGE_NAME:$IMAGE_TAG" - FROM --platform="linux/$TARGETARCH" scratch - COPY --platform="linux/$TARGETARCH" +binary/* / + # Distroless is used instead of `scratch` as root CA certs are required. + FROM --platform="linux/$GOARCH" gcr.io/distroless/static-debian12 + COPY (+binary/* --GOOS="linux" --GOARCH="$GOARCH") / # Unfortunately arg expansion is not supported here, see https://github.com/earthly/earthly/issues/1846 ENTRYPOINT [ "/ami-cleanup" ] # Process the outputs - SAVE IMAGE --push "$CONTAINER_REGISTRY$IMAGE_NAME:$IMAGE_TAG" + SAVE IMAGE --push "$OCI_REGISTRY$IMAGE_NAME:$IMAGE_TAG" # Same as `binary`, but wraps the output in a tarball. tarball: - ARG --required TARGETOS - ARG --required TARGETARCH + ARG GOOS=$USEROS + ARG GOARCH=$USERARCH ARG --required NATIVEARCH - ARG TARBALL_NAME="$BINARY_NAME-$TARGETOS-$TARGETARCH.tar.gz" + ARG TARBALL_NAME="$BINARY_NAME-$GOOS-$GOARCH.tar.gz" FROM --platform="linux/$NATIVEARCH" alpine:3.19.0 WORKDIR /tarball - COPY --platform="$TARGETOS/$TARGETARCH" +binary/* . + COPY (+binary/* --GOOS="$GOOS" --GOARCH="$GOARCH") . RUN tar -czvf "$TARBALL_NAME" * - SAVE ARTIFACT $TARBALL_NAME AS LOCAL outputs/$TARGETOS/$TARGETARCH/$TARBALL_NAME - -local-tarball: - ARG --required USERPLATFORM - BUILD --platform="$USERPLATFORM" +tarball + SAVE ARTIFACT $TARBALL_NAME AS LOCAL outputs/$GOOS/$GOARCH/$TARBALL_NAME all: - BUILD +local-binary - BUILD +local-tarball + BUILD +binary + BUILD +tarball BUILD +container-image # Runs the project's Go tests. test: - # Probably not needed, but this supports running tests on different architectures. - ARG --required TARGETPLATFORM # For options, see # https://github.com/gotestyourself/gotestsum?tab=readme-ov-file#output-format ARG OUTPUT_FORMAT="pkgname-and-test-fails" - FROM --platform "$TARGETPLATFORM" +go-environment + FROM +go-environment WORKDIR /go/src - CACHE --sharing shared $(go env GOMODCACHE) - CACHE --sharing shared --id gocache $(go env GOCACHE) RUN go install gotest.tools/gotestsum@latest + COPY src/ . RUN gotestsum --format "$OUTPUT_FORMAT" ./... -- -shuffle on -timeout 2m -race lint: @@ -137,13 +135,10 @@ lint: # Setup the linter and configure the environment FROM +go-environment WORKDIR /go/src - ENV GOLANGCI_LINT_CACHE=/golangci-lint-cache - CACHE $GOLANGCI_LINT_CACHE - CACHE --sharing shared $(go env GOMODCACHE) - CACHE --sharing shared --id gocache $(go env GOCACHE) RUN go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.2 # Run the linter + COPY src/ . RUN golangci-lint run ./... --out-format "$OUTPUT_FORMAT" # Removes local file and container image artifacts. @@ -158,20 +153,121 @@ clean: RUN docker image rm --force "$IMAGE" END +# Environment with the "chan" tool for operating on changelogs, along with the changelog itself +changelog-environment: + ARG NATIVEARCH + FROM --platform="linux/$NATIVEARCH" node:21.6-alpine3.18 + CACHE --sharing shared --id npm $(echo "$HOME/.npm") + RUN npm install --global '@geut/chan@3.2.9' + WORKDIR /changelog + COPY CHANGELOG.md . + +build-release-changelog: + ARG --required GIT_TAG + FROM +changelog-environment + DO +EXTRACT_VERSION --GIT_TAG=$GIT_TAG + + # Set the correct prerelease-dependent flag for updating the changelog + IF [ "${GIT_VERSION#*-}" != "$GIT_VERSION" ] + LET FLAGS="--allow-prerelease" + ELSE + LET FLAGS="--merge-prerelease" + END + + # Get a list of the changes, and + RUN CH_OUTPUT=$( \ + chan release \ + --git-compare-template "https://github.com/$REPO_NAME/compare/[prev]...[next]" \ + --git-release-template "httpx://github.com/$REPO_NAME/releases/tag/[next]" \ + $FLAGS \ + "$GIT_VERSION" \ + ) && \ + # Check if the release had any changes + echo "$CH_OUTPUT" | grep -qv "not new" || ( \ + echo -e "\n\n\nChangelog contains no unreleased changes, aborting\n\n\n"; exit 1 \ + ) + SAVE ARTIFACT CHANGELOG.md AS LOCAL CHANGELOG.md + +# Target to file a release PR, should be ran locally +create-release-pr: + ARG --required GIT_TAG + + LOCALLY + IF ! command -v git + RUN echo "Missing \`git\` command locally"; exit 1 + END + IF ! command -v gh + RUN echo "Missing \`gh\` Github CLI command locally"; exit 1 + END + + # Create a new release branch + RUN git fetch origin && \ + git checkout main && \ + git pull && \ + git checkout -b "release/$GIT_TAG" main && \ + git checkout . + + # Update the changelog + COPY +build-release-changelog/CHANGELOG.md . + + # Push the changes, file a PR, and push a new tag + DO +EXTRACT_VERSION --GIT_TAG=$GIT_TAG + RUN git add CHANGELOG.md && \ + git commit -m "Release $PROJECT_NAME $GIT_VERSION ($GIT_TAG)" && \ + git push origin && \ + PR_URL=$(gh pr create --fill --base "main" --reviewer "@me" --assignee "@me") && \ + echo "PR: $PR_URL" && \ + open --url "$PR_URL" && \ + while [ "$(gh pr view "$PR_URL" --json 'state' -q '.state')" != "MERGED" ]; do \ + echo "Waiting for PR to merge..."; \ + sleep 60; \ + done && \ + echo "PR merged, cutting release" && \ + git fetch origin && \ + git checkout main && \ + git pull && \ + git tag "$GIT_TAG" && \ + git push origin --tags && \ + sleep 5 && \ # Naively wait 5s to allow time for the run to be queued + gh run list --workflow cd.yaml --event push --branch "$GIT_TAG" && \ + open --url "$(gh run list --workflow ${PROJECT_NAME}-cd.yaml --event push --branch $GIT_TAG --json 'url' --jq '. | first | .url')" + # Cuts a new GH release and pushes file assets to it. Also pushes container images. release: ARG --required GIT_TAG # This global var is redeclared here to ensure that it is set via `--required` - ARG CONTAINER_REGISTRY="ghcr.io/gravitational/shared-workflows/" + ARG OCI_REGISTRY="ghcr.io/gravitational/" + ARG EARTHLY_PUSH + ARG NATIVEARCH + + # Validate the changelog and get release notes + FROM +changelog-environment + DO +EXTRACT_VERSION --GIT_TAG=$GIT_TAG + IF grep -qE "## \\[?${GIT_VERSION#v}\\]? - " CHANGELOG.md + LET CHANGELOG_ENTRIES=$(chan show "$GIT_VERSION") + END + + IF $EARTHLY_PUSH && [ -z "$CHANGELOG_ENTRIES" ] + RUN echo "No changelog entry detected for $GIT_VERSION, aborting"; exit 1 + END # Create GH release and upload artifact(s) - FROM alpine:3.19.0 + FROM --platform="linux/$NATIVEARCH" alpine:3.19.0 + # Unfortunately GH does not release a container image for their CLI, see https://github.com/cli/cli/issues/2027 RUN apk add github-cli WORKDIR /release_artifacts - COPY --platform=linux/amd64 --platform=linux/arm64 --platform=darwin/arm64 +tarball/* . - COPY CHANGELOG.md /CHANGELOG.md + COPY (+tarball/* --GOOS=linux --GOARCH=amd64) (+tarball/* --GOOS=linux --GOARCH=arm64) (+tarball/* --GOOS=darwin --GOARCH=arm64) . + + # Determine if the prerelease flag should be set + IF [ "${GIT_VERSION#*-}" != "$GIT_VERSION" ] + LET PRERELEASE_FLAG="--prerelease" + END + # Run commands with "--push" set will only run when the "--push" arg is provided via CLI - RUN --push gh release create --draft --verify-tag --notes-file "/CHANGELOG.md" --prerelease "$GIT_TAG" "./*" + RUN --push --secret GH_TOKEN \ + gh release create \ + --title "gha-exporter $GIT_VERSION" --verify-tag --notes "$CHANGELOG_ENTRIES" $PRERELEASE_FLAG "$GIT_TAG" --repo "$REPO_NAME" \ + ./* # Build container images and push them - BUILD --platform=linux/amd64 --platform=linux/arm64 +container-image --CONTAINER_REGISTRY="$CONTAINER_REGISTRY" + BUILD --platform=linux/amd64 --platform=linux/arm64 +container-image