From b4dde60fcaf423a0f3842ed98f7c9324917d22ff Mon Sep 17 00:00:00 2001 From: Jimmi Dyson Date: Mon, 16 Oct 2023 14:14:40 +0100 Subject: [PATCH] build: Use ko for image building --- .github/workflows/checks.yml | 5 -- .github/workflows/release-tag.yaml | 7 --- .go-tools | 1 - .goreleaser.yml | 81 +++++++++++---------------- .tool-versions | 10 ---- devbox.json | 1 + devbox.lock | 22 ++++++-- hack/flakes/go-mod-upgrade/flake.lock | 44 +++++++++++++++ hack/flakes/go-mod-upgrade/flake.nix | 64 +++++++++++++++++++++ make/all.mk | 3 - make/ci.mk | 50 ----------------- make/go.mk | 4 +- make/goreleaser.mk | 2 +- make/tools.mk | 43 -------------- make/upx.mk | 20 ------- 15 files changed, 163 insertions(+), 194 deletions(-) delete mode 100644 .go-tools delete mode 100644 .tool-versions create mode 100644 hack/flakes/go-mod-upgrade/flake.lock create mode 100644 hack/flakes/go-mod-upgrade/flake.nix delete mode 100644 make/ci.mk delete mode 100644 make/tools.mk delete mode 100644 make/upx.mk diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index de477451..5cbd5c39 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -57,11 +57,6 @@ jobs: with: enable-cache: true - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - with: - platforms: arm64 - - name: Run e2e tests run: devbox run -- make e2e-test env: diff --git a/.github/workflows/release-tag.yaml b/.github/workflows/release-tag.yaml index 2806cedb..50d61f37 100644 --- a/.github/workflows/release-tag.yaml +++ b/.github/workflows/release-tag.yaml @@ -27,13 +27,6 @@ jobs: with: enable-cache: true - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Release run: devbox run -- make release env: diff --git a/.go-tools b/.go-tools deleted file mode 100644 index e9360f06..00000000 --- a/.go-tools +++ /dev/null @@ -1 +0,0 @@ -github.com/oligot/go-mod-upgrade@v0.9.1 diff --git a/.goreleaser.yml b/.goreleaser.yml index 99510dd4..c4908583 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -47,9 +47,21 @@ builds: - amd64 - arm64 mod_timestamp: '{{ .CommitTimestamp }}' - hooks: - post: - - cmd: make SKIP_UPX={{ if index .Env "SKIP_UPX" }}{{ .Env.SKIP_UPX }}{{ else }}{{ .IsSnapshot }}{{ end }} GOOS={{ .Os }} GOARCH={{ .Arch }} UPX_TARGET={{ .Path }} upx + +upx: + - enabled: "{{ not .IsSnapshot }}" + goos: + - linux + compress: "9" + lzma: true + - enabled: "{{ not .IsSnapshot }}" + goos: + - windows + goarch: + - amd64 + compress: "9" + lzma: true + archives: - name_template: '{{ .ProjectName }}_v{{trimprefix .Version "v"}}_{{ .Os }}_{{ .Arch }}' format_overrides: @@ -58,51 +70,24 @@ archives: builds: - mindthegap rlcp: true -dockers: - - image_templates: - # Specify the image tag including `-amd64` suffix if the build is not a snapshot build or is not being built on - # arm64 machine. This allows for using the snapshot image build without the archtecture specific suffix - # consistently on local machines, i.e. can always use `ghcr.io/mesosphere/mindthegap:v` on the machine the snapshot - # is built on. - # - # For a release build the `-amd64` suffix will always be included and the `docker_manifests` specification below - # will create the final multiplatform manifest to be pushed to the registry. - - 'ghcr.io/mesosphere/mindthegap:v{{trimprefix .Version "v"}}{{ if or (not .IsSnapshot) (not (eq .Runtime.Goarch "amd64")) }}-amd64{{ end }}' - use: buildx - dockerfile: Dockerfile - build_flag_templates: - - "--platform=linux/amd64" - - "--pull" - - "--label=org.opencontainers.image.created={{.Date}}" - - "--label=org.opencontainers.image.title=mindthegap" - - "--label=org.opencontainers.image.revision={{.FullCommit}}" - - "--label=org.opencontainers.image.version={{.Version}}" - - "--label=org.opencontainers.image.source={{.GitURL}}" - - image_templates: - # Specify the image tag including `-amd64v8` suffix if the build is not a snapshot build or is not being built on - # arm64 machine. This allows for using the snapshot image build without the archtecture specific suffix - # consistently on local machines, i.e. can always use `ghcr.io/mesosphere/mindthegap:v` on the machine the snapshot - # is built on. - # - # For a release build the `-amd64v8` suffix will always be included and the `docker_manifests` specification below - # will create the final multiplatform manifest to be pushed to the registry. - - 'ghcr.io/mesosphere/mindthegap:v{{trimprefix .Version "v"}}{{ if or (not .IsSnapshot) (not (eq .Runtime.Goarch "arm64")) }}-arm64v8{{ end }}' - use: buildx - goarch: arm64 - dockerfile: Dockerfile - build_flag_templates: - - "--platform=linux/arm64" - - "--pull" - - "--label=org.opencontainers.image.created={{.Date}}" - - "--label=org.opencontainers.image.title=mindthegap" - - "--label=org.opencontainers.image.revision={{.FullCommit}}" - - "--label=org.opencontainers.image.version={{.Version}}" - - "--label=org.opencontainers.image.source={{.GitURL}}" -docker_manifests: - - name_template: ghcr.io/mesosphere/mindthegap:v{{trimprefix .Version "v"}} - image_templates: - - ghcr.io/mesosphere/mindthegap:v{{trimprefix .Version "v"}}-amd64 - - ghcr.io/mesosphere/mindthegap:v{{trimprefix .Version "v"}}-arm64v8 + +kos: + - id: mindthegap + build: mindthegap + labels: + org.opencontainers.image.created: "{{ .CommitDate }}" + org.opencontainers.image.title: mindthegap + org.opencontainers.image.revision: "{{ .FullCommit }}" + org.opencontainers.image.version: v{{trimprefix .Version "v"}} + org.opencontainers.image.source: "{{ .GitURL }}" + platforms: + - linux/amd64 + - linux/arm64 + repository: ghcr.io/mesosphere/mindthegap + bare: true + tags: + - v{{trimprefix .Version "v"}} + checksum: name_template: 'checksums.txt' snapshot: diff --git a/.tool-versions b/.tool-versions deleted file mode 100644 index 428ce1e1..00000000 --- a/.tool-versions +++ /dev/null @@ -1,10 +0,0 @@ -gcloud 433.0.1 -ginkgo 2.9.7 -gojq 0.12.13 -golang 1.20.4 -golangci-lint 1.53.2 -goreleaser 1.18.2 -helm 3.12.0 -pre-commit 3.3.2 -shfmt 3.6.0 -upx 4.0.2 diff --git a/devbox.json b/devbox.json index 072f076e..e975a3b3 100644 --- a/devbox.json +++ b/devbox.json @@ -7,6 +7,7 @@ "findutils@latest", "ginkgo@latest", "git@latest", + "path:./hack/flakes/go-mod-upgrade", "gnused@latest", "gnugrep@latest", "gnumake@latest", diff --git a/devbox.lock b/devbox.lock index c067dee6..bb673188 100644 --- a/devbox.lock +++ b/devbox.lock @@ -80,10 +80,24 @@ "version": "0.11.0" }, "goreleaser@latest": { - "last_modified": "2023-07-23T03:35:12Z", - "resolved": "github:NixOS/nixpkgs/af8cd5ded7735ca1df1a1174864daab75feeb64a#goreleaser", - "source": "devbox-search", - "version": "1.19.2" + "last_modified": "2023-10-01T23:11:39Z", + "resolved": "github:NixOS/nixpkgs/f8be2c43b4b1c1aec02f741ae909696b8410ebc1#goreleaser", + "source": "devbox-search", + "version": "1.21.2", + "systems": { + "aarch64-darwin": { + "store_path": "/nix/store/x9zmzrlmgwrvh8873yj4gyfwirzmr56q-goreleaser-1.21.2" + }, + "aarch64-linux": { + "store_path": "/nix/store/p33a0pj50ajjhqr5af7p3gz9m4x6kxps-goreleaser-1.21.2" + }, + "x86_64-darwin": { + "store_path": "/nix/store/bcxrzsd200xxj50y3rm214brig4g4gbb-goreleaser-1.21.2" + }, + "x86_64-linux": { + "store_path": "/nix/store/qn65854flb2l2kb0168c3ci24ra0irm6-goreleaser-1.21.2" + } + } }, "gotestsum@latest": { "last_modified": "2023-07-23T03:35:12Z", diff --git a/hack/flakes/go-mod-upgrade/flake.lock b/hack/flakes/go-mod-upgrade/flake.lock new file mode 100644 index 00000000..6a4693ca --- /dev/null +++ b/hack/flakes/go-mod-upgrade/flake.lock @@ -0,0 +1,44 @@ +{ + "nodes": { + "nixpkgs": { + "locked": { + "lastModified": 1693663421, + "narHash": "sha256-ImMIlWE/idjcZAfxKK8sQA7A1Gi/O58u5/CJA+mxvl8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "e56990880811a451abd32515698c712788be5720", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "utils": "utils" + } + }, + "utils": { + "locked": { + "lastModified": 1652776076, + "narHash": "sha256-gzTw/v1vj4dOVbpBSJX4J0DwUR6LIyXo7/SuuTJp1kM=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "04c1b180862888302ddfb2e3ad9eaa63afc60cf8", + "type": "github" + }, + "original": { + "owner": "numtide", + "ref": "v1.0.0", + "repo": "flake-utils", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/hack/flakes/go-mod-upgrade/flake.nix b/hack/flakes/go-mod-upgrade/flake.nix new file mode 100644 index 00000000..9aaf768c --- /dev/null +++ b/hack/flakes/go-mod-upgrade/flake.nix @@ -0,0 +1,64 @@ +# Copyright 2023 D2iQ, Inc. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ + description = "Update outdated Go dependencies interactively"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + utils.url = "github:numtide/flake-utils/v1.0.0"; + }; + + outputs = + { self + , nixpkgs + , utils + , + }: + let + appReleaseVersion = "0.9.1"; + appReleaseBinaries = { + "x86_64-linux" = { + fileName = "go-mod-upgrade_${appReleaseVersion}_Linux_x86_64.tar.gz"; + sha256 = "38b7f36b275fa08bedf0e4c7fb1eaf256fa632a7489abe7c40a1d2b87a688b01"; + }; + "x86_64-darwin" = { + fileName = "go-mod-upgrade_${appReleaseVersion}_Darwin_x86_64.tar.gz"; + sha256 = "e1e0294040cfadde0f119590f37fbff73654abc482ac60c1e3ca60b867326713"; + }; + "aarch64-darwin" = { + fileName = "go-mod-upgrade_${appReleaseVersion}_Darwin_arm64.tar.gz"; + sha256 = "15027f435a85f31346fd0796977180c43c737b7fe7bbb4fc3bcc5f4b8f32804c"; + }; + }; + supportedSystems = builtins.attrNames appReleaseBinaries; + in + utils.lib.eachSystem supportedSystems (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + appReleaseBinary = appReleaseBinaries.${system}; + in + rec { + packages.go-mod-upgrade = pkgs.stdenv.mkDerivation { + pname = "go-mod-upgrade"; + version = appReleaseVersion; + + src = pkgs.fetchurl { + url = "https://github.com/oligot/go-mod-upgrade/releases/download/v${appReleaseVersion}/${appReleaseBinary.fileName}"; + sha256 = appReleaseBinary.sha256; + }; + + sourceRoot = "."; + + installPhase = '' + install -m755 -D go-mod-upgrade $out/bin/go-mod-upgrade + ''; + }; + packages.default = packages.go-mod-upgrade; + + apps.go-mod-upgrade = utils.lib.mkApp { + drv = packages.go-mod-upgrade; + }; + apps.default = apps.go-mod-upgrade; + }); +} diff --git a/make/all.mk b/make/all.mk index ed4da7fb..6690697e 100644 --- a/make/all.mk +++ b/make/all.mk @@ -8,11 +8,8 @@ include $(INCLUDE_DIR)shell.mk include $(INCLUDE_DIR)help.mk include $(INCLUDE_DIR)repo.mk include $(INCLUDE_DIR)platform.mk -include $(INCLUDE_DIR)tools.mk include $(INCLUDE_DIR)pre-commit.mk include $(INCLUDE_DIR)go.mk include $(INCLUDE_DIR)goreleaser.mk include $(INCLUDE_DIR)docker.mk -include $(INCLUDE_DIR)ci.mk include $(INCLUDE_DIR)tag.mk -include $(INCLUDE_DIR)upx.mk diff --git a/make/ci.mk b/make/ci.mk deleted file mode 100644 index 63d7cd20..00000000 --- a/make/ci.mk +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright 2021-2023 D2iQ, Inc. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -CI_DOCKERFILE ?= $(REPO_ROOT)/Dockerfile.ci - -ifneq ($(wildcard $(CI_DOCKERFILE)),) -CI_DOCKER_TAG ?= $(shell (cat $(CI_DOCKERFILE) \ - $(if $(CI_DOCKER_BUILD_ARGS),&& echo $(CI_DOCKER_BUILD_ARGS))) \ - | shasum | awk '{ print $$1 }') -CI_DOCKER_IMG ?= $(GITHUB_ORG)/$(GITHUB_REPOSITORY)-ci:$(CI_DOCKER_TAG) - -.PHONY: ci.docker.ensure -ci.docker.ensure: ## Ensures the docker image is locally available -ci.docker.ensure: dockerauth ; $(info $(M) Ensuring CI Docker image is available locally) - (docker image inspect $(CI_DOCKER_IMG) &>/dev/null && echo '$(CI_DOCKER_IMG) already exists - skipping image build' ) || \ - docker pull $(CI_DOCKER_IMG) || \ - $(MAKE) ci.docker.build - -.PHONY: ci.docker.build -ci.docker.build: ## Builds the CI Docker image -ci.docker.build: dockerauth ; $(info $(M) Building CI Docker image) - DOCKER_BUILDKIT=1 docker build \ - --tag $(CI_DOCKER_IMG) \ - $(if $(CI_DOCKER_BUILD_ARGS),$(addprefix --build-arg ,$(CI_DOCKER_BUILD_ARGS))) \ - -f $(CI_DOCKERFILE) . - -.PHONY: ci.docker.push -ci.docker.push: ## Pushes the CI Docker image -ci.docker.push: ci.docker.ensure ; $(info $(M) Pushes the CI Docker image) - docker push $(CI_DOCKER_IMG) - -.PHONY: ci.docker.run -ci.docker.run: ## Runs the build in the CI Docker image. -ci.docker.run: RUN_WHAT ?= -ci.docker.run: ci.docker.ensure ; $(info $(M) Runs the build in the CI Docker image) - @docker run --rm -i$(if $(RUN_WHAT),,$(if $(INTERACTIVE),t)) \ - -u $(shell id -u):$(shell id -g) \ - -v $(REPO_ROOT):$(REPO_ROOT) \ - -w $(REPO_ROOT) \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /etc/docker/certs.d:/etc/docker/certs.d \ - $(if $(DOCKER_USERNAME),-e DOCKER_USERNAME=$(DOCKER_USERNAME)) \ - $(if $(DOCKER_PASSWORD),-e DOCKER_PASSWORD=$(DOCKER_PASSWORD)) \ - $(if $(SSH_AUTH_SOCK),-v $(SSH_AUTH_SOCK):$(SSH_AUTH_SOCK) -e SSH_AUTH_SOCK=$(SSH_AUTH_SOCK)) \ - $(if $(GITHUB_USER_TOKEN),-e GITHUB_USER_TOKEN=$(GITHUB_USER_TOKEN) -e GITHUB_TOKEN=$(GITHUB_USER_TOKEN),$(if $(GITHUB_TOKEN),-e GITHUB_TOKEN=$(GITHUB_TOKEN))) \ - --net=host \ - $(CI_DOCKER_IMG) \ - $(RUN_WHAT) - -endif diff --git a/make/go.mk b/make/go.mk index ea0747b4..93ce3ae7 100644 --- a/make/go.mk +++ b/make/go.mk @@ -79,7 +79,7 @@ E2E_FLAKE_ATTEMPTS ?= 1 e2e-test: ## Runs e2e tests $(info $(M) running e2e tests$(if $(E2E_LABEL), labelled "$(E2E_LABEL)")$(if $(E2E_FOCUS), matching "$(E2E_FOCUS)")) ifndef E2E_SKIP_BUILD - $(MAKE) GORELEASER_FLAGS=$$'--config=<(env GOOS=$(shell go env GOOS) GOARCH=$(shell go env GOARCH) gojq --yaml-input --yaml-output \'del(.builds[0].goarch) | del(.builds[0].goos) | .builds[0].targets|=(["linux_amd64","linux_arm64",env.GOOS+"_"+env.GOARCH] | unique | map(. | sub("_amd64";"_amd64_v1")))\' .goreleaser.yml) --clean --skip-validate --skip-publish' release + $(MAKE) GORELEASER_FLAGS=$$'--config=<(env GOOS=$(shell go env GOOS) GOARCH=$(shell go env GOARCH) gojq --yaml-input --yaml-output \'del(.builds[0].goarch) | del(.builds[0].goos) | .builds[0].targets|=(["linux_amd64","linux_arm64",env.GOOS+"_"+env.GOARCH] | unique | map(. | sub("_amd64";"_amd64_v1")))\' .goreleaser.yml) --clean --skip=validate,publish' release endif ginkgo run \ --r \ @@ -164,5 +164,5 @@ go-generate: ; $(info $(M) running go generate) .PHONY: go-mod-upgrade go-mod-upgrade: ## Interactive check for direct module dependency upgrades -go-mod-upgrade: install-tool.go.go-mod-upgrade ; $(info $(M) checking for direct module dependency upgrades) +go-mod-upgrade: ; $(info $(M) checking for direct module dependency upgrades) go-mod-upgrade diff --git a/make/goreleaser.mk b/make/goreleaser.mk index 6a5de3bb..202c0aef 100644 --- a/make/goreleaser.mk +++ b/make/goreleaser.mk @@ -35,7 +35,7 @@ release-snapshot: dockerauth; $(info $(M) building snapshot release $*) goreleaser --debug=$(GORELEASER_DEBUG) \ release \ --snapshot \ - --skip-publish \ + --skip=publish \ --clean \ --parallelism=$(GORELEASER_PARALLELISM) \ --timeout=60m diff --git a/make/tools.mk b/make/tools.mk deleted file mode 100644 index 4c4083b8..00000000 --- a/make/tools.mk +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2021-2023 D2iQ, Inc. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Override this in your own top-level Makefile if this is in a different path in your repo. -GO_TOOLS_FILE ?= $(REPO_ROOT)/.go-tools - -# Explicitly override GOBIN so it does not inherit from the environment - this allows for a truly -# self-contained build environment for the project. -override GOBIN := $(REPO_ROOT)/.local/bin -export GOBIN -export PATH := $(GOBIN):$(PATH) - -ifneq ($(wildcard $(GO_TOOLS_FILE)),) -define install_go_tool - mkdir -p $(GOBIN) - CGO_ENABLED=0 go install -v $$(grep -Eo '^.+$1[^ ]+' $(GO_TOOLS_FILE)) -endef - -.PHONY: -install-tool.go.%: ## Installs go tools -install-tool.go.%: ; $(info $(M) installing go tool $*) - $(call install_go_tool,$*) -endif - -.PHONY: upgrade-go-tools -upgrade-go-tools: ## Upgrades all go tools to latest available versions -upgrade-go-tools: ; $(info $(M) upgrading all go tools to latest available versions) - grep -v '# FREEZE' .go-tools | \ - grep -Eo '^[^#][^@]+' | \ - xargs -I {} bash -ec ' \ - original_module_path={}; \ - module_path={}; \ - while [ "$${module_path}" != "." ]; do \ - LATEST_VERSION=$$(go list -m $${module_path}@latest 2>/dev/null || echo ""); \ - if [ -n "$${LATEST_VERSION}" ]; then \ - sed -i "s|$${original_module_path}@.\+$$|$${original_module_path}@$${LATEST_VERSION#* }|" .go-tools; \ - exit; \ - else \ - module_path=$$(dirname $${module_path}); \ - fi; \ - done; \ - echo "Failed to find latest module version for $${original_module_path}"; \ - exit 1' diff --git a/make/upx.mk b/make/upx.mk deleted file mode 100644 index 3a38fa61..00000000 --- a/make/upx.mk +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2021-2023 D2iQ, Inc. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -SKIP_UPX := true - -.PHONY: upx -upx: UPX_REAL_TARGET := $(addsuffix $(if $(filter $(GOOS),windows),.exe),$(basename $(UPX_TARGET))) -ifneq ($(SKIP_UPX),true) -ifeq ($(GOOS)/$(GOARCH),windows/arm64) -upx: ; $(info $(M) skipping packing $(UPX_REAL_TARGET) - $(GOOS)/$(GOARCH) is not yet supported by upx) -else ifeq ($(GOOS),darwin) -upx: ; $(info $(M) skipping packing $(UPX_REAL_TARGET) - $(GOOS) has a bug in packing especially on Ventura - https://github.com/upx/upx/issues/612) -else -upx: ## Pack executable using upx -upx: ; $(info $(M) packing $(UPX_REAL_TARGET)) - (upx -l $(UPX_REAL_TARGET) &>/dev/null && echo $(UPX_REAL_TARGET) is already packed) || upx -9 --lzma $(UPX_REAL_TARGET) -# Double check file is successfully compressed - seen errors with macos binaries - upx -t $(UPX_REAL_TARGET) &>/dev/null || (echo $(UPX_REAL_TARGET) is broken after upx compression && exit 1) -endif -endif