From 8a099acdec6bb00a64442ecba8ebbfe7d8faf91f Mon Sep 17 00:00:00 2001 From: Salah Al Saleh Date: Tue, 12 Dec 2023 13:28:37 -0800 Subject: [PATCH] build kotsadm-migrations, rqlite, minio, and dex with apko and melange (#4210) * build kotsadm-migrations, rqlite, minio, and dex with apko and melange --- .../build-push-image-with-apko/action.yml | 45 +++++++ .../action.yml | 62 +++++++++ .github/workflows/alpha.yaml | 69 +++++++--- .github/workflows/build-test.yaml | 46 +++---- .github/workflows/regression.yaml | 9 +- .github/workflows/release.yaml | 120 +++++++++++++----- .image.env | 6 +- Makefile | 27 ++-- cmd/imagedeps/README.md | 20 +-- cmd/imagedeps/image-spec | 6 +- cmd/imagedeps/main.go | 24 +++- cmd/imagedeps/main_test.go | 79 +++++------- cmd/imagedeps/tag-finder.go | 57 ++++++++- cmd/imagedeps/testdata/basic/input-spec | 1 - .../{filter-github => dex}/.image.env | 2 +- .../{filter-github => dex}/constants.go | 2 +- cmd/imagedeps/testdata/dex/input-spec | 1 + .../testdata/filter-github/input-spec | 1 - .../testdata/{basic => minio}/.image.env | 2 +- .../testdata/{basic => minio}/constants.go | 2 +- cmd/imagedeps/testdata/minio/input-spec | 1 + cmd/imagedeps/testdata/rqlite/.image.env | 2 +- cmd/imagedeps/testdata/rqlite/constants.go | 2 +- cmd/imagedeps/testdata/rqlite/input-spec | 2 +- cmd/imagedeps/testdata/schemahero/.image.env | 2 +- .../testdata/schemahero/constants.go | 2 +- cmd/imagedeps/testdata/schemahero/input-spec | 2 +- .../replacers/expected/test.Dockerfile | 0 .../replacers/expected/test.mk | 0 .../replacers/input/test.Dockerfile | 0 .../replacers/input/test.mk | 0 .../testdata/with-overrides/.image.env | 5 - .../testdata/with-overrides/constants.go | 10 -- .../testdata/with-overrides/input-spec | 2 - deploy/apko.yaml.tmpl | 4 - deploy/apko_melange_build.md | 4 +- deploy/dex.Dockerfile | 2 - deploy/dex/apko.yaml | 26 ++++ deploy/kurl/kotsadm/template/base/Manifest | 2 +- deploy/kurl/kotsadm/template/base/rqlite.yaml | 4 +- deploy/kurl/kotsadm/template/generate.sh | 3 +- deploy/minio/apko.yaml | 24 ++++ deploy/rqlite/apko.yaml | 26 ++++ kurl_proxy/deploy/apko.yaml.tmpl | 4 - migrations/Makefile | 16 +-- migrations/deploy/apko.yaml.tmpl | 32 +++++ migrations/deploy/melange.yaml.tmpl | 32 +++++ migrations/kustomize/overlays/dev/rqlite.yaml | 4 +- .../kustomize/overlays/okteto/rqlite.yaml | 4 +- pkg/identity/deploy/deploy.go | 2 +- pkg/image/constants.go | 8 +- pkg/kotsadm/minio.go | 10 +- pkg/kotsadm/minio_test.go | 7 + pkg/kotsadm/objects/images.go | 9 +- pkg/kotsadm/objects/minio_objects.go | 2 +- pkg/kotsadm/objects/rqlite_objects.go | 4 + .../objects/scripts/import-minio-data.sh | 2 +- pkg/snapshot/filesystem_minio.go | 7 +- 58 files changed, 594 insertions(+), 255 deletions(-) create mode 100644 .github/actions/build-push-image-with-apko/action.yml create mode 100644 .github/actions/build-push-kotsadm-migrations-image/action.yml delete mode 100644 cmd/imagedeps/testdata/basic/input-spec rename cmd/imagedeps/testdata/{filter-github => dex}/.image.env (83%) rename cmd/imagedeps/testdata/{filter-github => dex}/constants.go (81%) create mode 100644 cmd/imagedeps/testdata/dex/input-spec delete mode 100644 cmd/imagedeps/testdata/filter-github/input-spec rename cmd/imagedeps/testdata/{basic => minio}/.image.env (83%) rename cmd/imagedeps/testdata/{basic => minio}/constants.go (81%) create mode 100644 cmd/imagedeps/testdata/minio/input-spec rename cmd/imagedeps/testdata/{with-overrides => schemahero}/replacers/expected/test.Dockerfile (100%) rename cmd/imagedeps/testdata/{with-overrides => schemahero}/replacers/expected/test.mk (100%) rename cmd/imagedeps/testdata/{with-overrides => schemahero}/replacers/input/test.Dockerfile (100%) rename cmd/imagedeps/testdata/{with-overrides => schemahero}/replacers/input/test.mk (100%) delete mode 100644 cmd/imagedeps/testdata/with-overrides/.image.env delete mode 100644 cmd/imagedeps/testdata/with-overrides/constants.go delete mode 100644 cmd/imagedeps/testdata/with-overrides/input-spec delete mode 100644 deploy/dex.Dockerfile create mode 100644 deploy/dex/apko.yaml create mode 100644 deploy/minio/apko.yaml create mode 100644 deploy/rqlite/apko.yaml create mode 100644 migrations/deploy/apko.yaml.tmpl create mode 100644 migrations/deploy/melange.yaml.tmpl diff --git a/.github/actions/build-push-image-with-apko/action.yml b/.github/actions/build-push-image-with-apko/action.yml new file mode 100644 index 0000000000..e2b474b56b --- /dev/null +++ b/.github/actions/build-push-image-with-apko/action.yml @@ -0,0 +1,45 @@ +name: 'Build and push images with apko' +description: 'Composite action for building and pushing images with apko' +inputs: + apko-config: + description: 'Path to apko config' + required: true + + image-name: + description: 'Full destination image name' + required: true + + registry-username: + description: 'Username to login to registry' + default: '' + required: false + + registry-password: + description: 'Password to login to registry' + default: '' + required: false + +runs: + using: "composite" + steps: + - id: check-image-exists + shell: bash + run: | + set -euo pipefail + if docker manifest inspect ${{ inputs.image-name }} > /dev/null 2>&1; then + echo "Image already exists. Will not overwrite." + echo "image-exists=true" >> "$GITHUB_OUTPUT" + else + echo "Image does not exist. Will build and push." + echo "image-exists=false" >> "$GITHUB_OUTPUT" + fi + + - uses: chainguard-images/actions/apko-publish@main + if: ${{ steps.check-image-exists.outputs.image-exists == 'false' }} + with: + config: ${{ inputs.apko-config }} + archs: x86_64 + tag: ${{ inputs.image-name }} + vcs-url: true + generic-user: ${{ inputs.registry-username }} + generic-pass: ${{ inputs.registry-password }} diff --git a/.github/actions/build-push-kotsadm-migrations-image/action.yml b/.github/actions/build-push-kotsadm-migrations-image/action.yml new file mode 100644 index 0000000000..858ac584db --- /dev/null +++ b/.github/actions/build-push-kotsadm-migrations-image/action.yml @@ -0,0 +1,62 @@ +name: 'Build and push kotsadm-migrations image' +description: 'Composite action for building and pushing kotsadm-migrations image' +inputs: + image-name: + description: 'Full destination kotsadm-migrations image name' + required: true + + git-tag: + description: 'Git tag' + required: true + + registry-username: + description: 'Username to login to registry' + default: '' + required: false + + registry-password: + description: 'Password to login to registry' + default: '' + required: false + +runs: + using: "composite" + steps: + - name: load environment variables from .image.env + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env + + - uses: shrink/actions-docker-extract@v3 + with: + image: schemahero/schemahero:${{ steps.dotenv.outputs.SCHEMAHERO_TAG }} + path: /schemahero + destination: migrations + + - name: template melange and apko configs + shell: bash + run: | + export GIT_TAG=${{ inputs.git-tag }} + envsubst '${GIT_TAG}' < migrations/deploy/melange.yaml.tmpl > migrations/deploy/melange.yaml + envsubst '${GIT_TAG}' < migrations/deploy/apko.yaml.tmpl > migrations/deploy/apko.yaml + + - id: cache-dir + shell: bash + run: echo "cache_dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" + + - uses: chainguard-dev/actions/melange-build@main + with: + config: migrations/deploy/melange.yaml + archs: x86_64 + sign-with-temporary-key: true + cache-dir: ${{ steps.cache-dir.outputs.cache_dir }} + + - uses: chainguard-images/actions/apko-publish@main + with: + config: migrations/deploy/apko.yaml + archs: x86_64 + tag: ${{ inputs.image-name }} + vcs-url: true + generic-user: ${{ inputs.registry-username }} + generic-pass: ${{ inputs.registry-password }} diff --git a/.github/workflows/alpha.yaml b/.github/workflows/alpha.yaml index 19d2d5c248..d06d5149c5 100644 --- a/.github/workflows/alpha.yaml +++ b/.github/workflows/alpha.yaml @@ -19,31 +19,60 @@ jobs: uses: ./.github/actions/version-tag - build-schema-migrations: + build-kotsadm-migrations: + runs-on: ubuntu-20.04 + needs: [generate-tag] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-push-kotsadm-migrations-image + with: + image-name: index.docker.io/kotsadm/kotsadm-migrations:alpha + git-tag: ${{ needs.generate-tag.outputs.tag }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + + + build-rqlite: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-push-image-with-apko + with: + apko-config: deploy/rqlite/apko.yaml + image-name: index.docker.io/kotsadm/rqlite:alpha + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + + + build-minio: runs-on: ubuntu-20.04 steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-push-image-with-apko + with: + apko-config: deploy/minio/apko.yaml + image-name: index.docker.io/kotsadm/minio:alpha + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} - - uses: azure/docker-login@v1 - env: - DOCKER_CONFIG: ./migrations/.docker - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Build - env: - DOCKER_CONFIG: ./.docker - run: | - mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C migrations schema-alpha + build-dex: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/build-push-image-with-apko + with: + apko-config: deploy/dex/apko.yaml + image-name: index.docker.io/kotsadm/dex:alpha + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} build-kotsadm: runs-on: ubuntu-20.04 needs: [generate-tag] permissions: - id-token: write # required to be able to assume the GCP SA identity to pull Chainguard packages. + id-token: write # required to be able to assume the GCP SA identity to pull private Chainguard packages. steps: - uses: actions/checkout@v4 - uses: ./.github/actions/build-push-kotsadm-image @@ -84,7 +113,7 @@ jobs: id: scan uses: aquasecurity/trivy-action@master with: - image-ref: "rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }}" + image-ref: "docker.io/kotsadm/rqlite:alpha" format: 'template' template: '@/contrib/sarif.tpl' output: 'rqlite-scan-output.sarif' @@ -111,7 +140,7 @@ jobs: id: scan uses: aquasecurity/trivy-action@master with: - image-ref: "minio/minio:${{ steps.dotenv.outputs.minio_tag }}" + image-ref: "docker.io/kotsadm/minio:alpha" format: 'template' template: '@/contrib/sarif.tpl' output: 'minio-scan-output.sarif' @@ -139,7 +168,7 @@ jobs: id: scan uses: aquasecurity/trivy-action@master with: - image-ref: "ghcr.io/dexidp/dex:${{ steps.dotenv.outputs.dex_tag }}" + image-ref: "docker.io/kotsadm/dex:alpha" format: 'template' template: '@/contrib/sarif.tpl' output: 'dex-scan-output.sarif' @@ -225,9 +254,9 @@ jobs: sarif_file: kotsadm-scan-output.sarif - scan_migrations: + scan_kotsadm_migrations: runs-on: ubuntu-20.04 - needs: [build-schema-migrations] + needs: [build-kotsadm-migrations] steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml index 17074b7728..80adb0f0f8 100644 --- a/.github/workflows/build-test.yaml +++ b/.github/workflows/build-test.yaml @@ -268,7 +268,7 @@ jobs: runs-on: ubuntu-20.04 needs: [ can-run-ci, generate-tag ] permissions: - id-token: write # required to be able to assume the GCP SA identity to pull Chainguard packages. + id-token: write # required to be able to assume the GCP SA identity to pull private Chainguard packages. steps: - uses: actions/setup-go@v4 with: @@ -383,7 +383,7 @@ jobs: build-migrations: runs-on: ubuntu-20.04 - needs: [ can-run-ci ] + needs: [ can-run-ci, generate-tag ] steps: - name: setup env run: | @@ -399,20 +399,10 @@ jobs: ref: ${{github.event.pull_request.head.ref}} repository: ${{github.event.pull_request.head.repo.full_name}} - - name: load environment variables from .image.env - uses: falti/dotenv-action@v1 - id: dotenv - with: - path: .image.env - - - name: build and push migrations for e2e - uses: docker/build-push-action@v5 + - uses: ./.github/actions/build-push-kotsadm-migrations-image with: - tags: ttl.sh/automated-${{ github.run_id }}/kotsadm-migrations:24h - context: ./migrations - file: ./migrations/deploy/Dockerfile - push: true - build-args: SCHEMAHERO_TAG=${{ steps.dotenv.outputs.schemahero_tag }} + image-name: ttl.sh/automated-${{ github.run_id }}/kotsadm-migrations:24h + git-tag: ${{ needs.generate-tag.outputs.tag }} push-minio: @@ -427,11 +417,10 @@ jobs: with: path: .image.env - - name: push minio for e2e - run: | - docker pull minio/minio:${{ steps.dotenv.outputs.minio_tag }} - docker tag minio/minio:${{ steps.dotenv.outputs.minio_tag }} ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }} - docker push ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.minio_tag }} + - uses: ./.github/actions/build-push-image-with-apko + with: + apko-config: deploy/minio/apko.yaml + image-name: ttl.sh/automated-${{ github.run_id }}/minio:${{ steps.dotenv.outputs.MINIO_TAG }} push-rqlite: @@ -446,11 +435,10 @@ jobs: with: path: .image.env - - name: push rqlite for CI - run: | - docker pull rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} - docker tag rqlite/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} - docker push ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} + - uses: ./.github/actions/build-push-image-with-apko + with: + apko-config: deploy/rqlite/apko.yaml + image-name: ttl.sh/automated-${{ github.run_id }}/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} push-dex: @@ -465,10 +453,10 @@ jobs: with: path: .image.env - - name: push dex for CI - run: | - docker build --pull -f deploy/dex.Dockerfile -t ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }} --build-arg TAG=${{ steps.dotenv.outputs.DEX_TAG }} . - docker push ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }} + - uses: ./.github/actions/build-push-image-with-apko + with: + apko-config: deploy/dex/apko.yaml + image-name: ttl.sh/automated-${{ github.run_id }}/dex:${{ steps.dotenv.outputs.DEX_TAG }} # only run validate-kurl-addon if changes to "deploy/kurl/kotsadm/template/**" diff --git a/.github/workflows/regression.yaml b/.github/workflows/regression.yaml index cb303f7a83..666823560f 100644 --- a/.github/workflows/regression.yaml +++ b/.github/workflows/regression.yaml @@ -90,15 +90,10 @@ jobs: with: name: kots path: ./automation/jumpbox/bin - - name: Download kotsadm image + - name: Download kotsadm release uses: actions/download-artifact@v3 with: - name: kotsadm-image - path: ./automation/jumpbox/docker-archive - - name: Download migrations - uses: actions/download-artifact@v3 - with: - name: migrations-image + name: kotsadm-release path: ./automation/jumpbox/docker-archive - name: Make kotsadm airgap archive with minio image working-directory: automation/jumpbox diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 86bcbcb582..daabb41492 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -62,28 +62,18 @@ jobs: echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" - build-schema-migrations: + build-kotsadm-migrations: runs-on: ubuntu-20.04 needs: [generate-tag] steps: - name: Checkout uses: actions/checkout@v4 - - uses: azure/docker-login@v1 - env: - DOCKER_CONFIG: ./migrations/.docker + - uses: ./.github/actions/build-push-kotsadm-migrations-image with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: "Release schema migrations on tag" - env: - GIT_TAG: ${{ needs.generate-tag.outputs.tag }} - DOCKER_CONFIG: ./.docker - run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make -C migrations schema-release - - name: Upload airgap image - uses: actions/upload-artifact@v3 - with: - name: migrations-image - path: ./migrations/bin/docker-archive + image-name: index.docker.io/kotsadm/kotsadm-migrations:${{ needs.generate-tag.outputs.tag }} + git-tag: ${{ needs.generate-tag.outputs.tag }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} build-web: runs-on: ubuntu-20.04 @@ -156,14 +146,72 @@ jobs: name: kots path: ./bin/kots + build-minio: + runs-on: ubuntu-20.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Read image tags from env file + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env + + - uses: ./.github/actions/build-push-image-with-apko + with: + apko-config: deploy/minio/apko.yaml + image-name: index.docker.io/kotsadm/minio:${{ steps.dotenv.outputs.MINIO_TAG }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + + build-rqlite: + runs-on: ubuntu-20.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Read image tags from env file + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env + + - uses: ./.github/actions/build-push-image-with-apko + with: + apko-config: deploy/rqlite/apko.yaml + image-name: index.docker.io/kotsadm/rqlite:${{ steps.dotenv.outputs.RQLITE_TAG }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + + build-dex: + runs-on: ubuntu-20.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Read image tags from env file + uses: falti/dotenv-action@v1 + id: dotenv + with: + path: .image.env + + - uses: ./.github/actions/build-push-image-with-apko + with: + apko-config: deploy/dex/apko.yaml + image-name: index.docker.io/kotsadm/dex:${{ steps.dotenv.outputs.DEX_TAG }} + registry-username: ${{ secrets.DOCKERHUB_USER }} + registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} + build-kotsadm: runs-on: ubuntu-20.04 needs: [generate-tag] permissions: - id-token: write # required to be able to assume the GCP SA identity to pull Chainguard packages. + id-token: write # required to be able to assume the GCP SA identity to pull private Chainguard packages. steps: - name: Checkout uses: actions/checkout@v4 + - uses: ./.github/actions/build-push-kotsadm-image with: chainguard-gcp-wif-pool: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }} @@ -173,21 +221,30 @@ jobs: git-tag: ${{ needs.generate-tag.outputs.tag }} registry-username: ${{ secrets.DOCKERHUB_USER }} registry-password: ${{ secrets.DOCKERHUB_PASSWORD }} - - uses: azure/docker-login@v1 - env: - DOCKER_CONFIG: ./.docker + + build-release: + runs-on: ubuntu-20.04 + needs: [generate-tag, build-kotsadm-migrations, build-kotsadm, build-minio, build-rqlite, build-dex] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Read image tags from env file + uses: falti/dotenv-action@v1 + id: dotenv with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + path: .image.env + - name: Build tagged release env: GIT_TAG: ${{ needs.generate-tag.outputs.tag }} DOCKER_CONFIG: ./.docker run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make build-release - - name: Upload airgap image + + - name: Upload kotsadm release uses: actions/upload-artifact@v3 with: - name: kotsadm-image + name: kotsadm-release path: ./bin/docker-archive goreleaser: @@ -250,7 +307,7 @@ jobs: generate-kurl-addon: runs-on: ubuntu-20.04 - needs: [ generate-tag, build-kurl-proxy, build-schema-migrations, build-kots, build-kotsadm ] + needs: [ generate-tag, build-kurl-proxy, build-kots, build-release ] outputs: addon_package_url: ${{ steps.addon-generate.outputs.addon_package_url }} env: @@ -359,17 +416,12 @@ jobs: build-airgap: runs-on: ubuntu-20.04 if: github.ref_type != 'branch' - needs: [build-kotsadm, goreleaser, build-schema-migrations, generate-tag] + needs: [goreleaser, generate-tag, build-release] steps: - - name: Download migrations - uses: actions/download-artifact@v3 - with: - name: migrations-image - path: ./docker-archive - - name: Download kotsadm image + - name: Download kotsadm release uses: actions/download-artifact@v3 with: - name: kotsadm-image + name: kotsadm-release path: ./docker-archive - name: Make kotsadm airgap archive with minio image run: | @@ -417,7 +469,7 @@ jobs: regression-test: if: github.ref_type == 'branch' - needs: [ regression-test-setup, generate-tag, build-kots, build-kotsadm, generate-kurl-addon ] + needs: [ regression-test-setup, generate-tag, build-kots, generate-kurl-addon, build-release ] uses: ./.github/workflows/regression.yaml with: version_tag_old: ${{ needs.regression-test-setup.outputs.last_release_tag }} diff --git a/.image.env b/.image.env index cb5fd3bde3..18d2f5d4e9 100644 --- a/.image.env +++ b/.image.env @@ -1,8 +1,8 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -MINIO_TAG='RELEASE.2023-11-11T08-14-41Z' +MINIO_TAG='0.20231101.183725' RQLITE_TAG='7.21.4' -DEX_TAG='v2.37.0' -SCHEMAHERO_TAG='0.16.0' +DEX_TAG='2.37.0' +SCHEMAHERO_TAG='0.17.0' LVP_TAG='v0.5.5' \ No newline at end of file diff --git a/Makefile b/Makefile index b2c6216431..7642f5ba30 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ include Makefile.build.mk CURRENT_USER := $(shell id -u -n) -MINIO_TAG ?= RELEASE.2023-11-11T08-14-41Z +MINIO_TAG ?= 0.20231101.183725 RQLITE_TAG ?= 7.21.4 -DEX_TAG ?= v2.37.0 +DEX_TAG ?= 2.37.0 LVP_TAG ?= v0.5.5 define sendMetrics @@ -118,31 +118,30 @@ build-ttl.sh: build all-ttl.sh: build-ttl.sh source .image.env && IMAGE=ttl.sh/${CURRENT_USER}/kotsadm-migrations:24h make -C migrations build_schema - docker pull minio/minio:${MINIO_TAG} - docker tag minio/minio:${MINIO_TAG} ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG} + docker pull kotsadm/minio:${MINIO_TAG} + docker tag kotsadm/minio:${MINIO_TAG} ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG} docker push ttl.sh/${CURRENT_USER}/minio:${MINIO_TAG} - docker pull rqlite/rqlite:${RQLITE_TAG} - docker tag rqlite/rqlite:${RQLITE_TAG} ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} + docker pull kotsadm/rqlite:${RQLITE_TAG} + docker tag kotsadm/rqlite:${RQLITE_TAG} ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} docker push ttl.sh/${CURRENT_USER}/rqlite:${RQLITE_TAG} -.PHONY: build-alpha -build-alpha: - docker build --pull -f deploy/Dockerfile --build-arg version=${GIT_TAG} -t kotsadm/kotsadm:alpha . - docker push kotsadm/kotsadm:alpha - .PHONY: build-release build-release: mkdir -p bin/docker-archive/kotsadm skopeo copy docker://kotsadm/kotsadm:${GIT_TAG} docker-archive:bin/docker-archive/kotsadm/${GIT_TAG} - docker build --pull -f deploy/dex.Dockerfile -t kotsadm/dex:${DEX_TAG} --build-arg TAG=${DEX_TAG} . - docker push kotsadm/dex:${DEX_TAG} + mkdir -p bin/docker-archive/kotsadm-migrations + skopeo copy docker://kotsadm/kotsadm-migrations:${GIT_TAG} docker-archive:bin/docker-archive/kotsadm-migrations/${GIT_TAG} + mkdir -p bin/docker-archive/dex skopeo copy docker://kotsadm/dex:${DEX_TAG} docker-archive:bin/docker-archive/dex/${DEX_TAG} mkdir -p bin/docker-archive/minio - skopeo copy docker://minio/minio:${MINIO_TAG} docker-archive:bin/docker-archive/minio/${MINIO_TAG} + skopeo copy docker://kotsadm/minio:${MINIO_TAG} docker-archive:bin/docker-archive/minio/${MINIO_TAG} + + mkdir -p bin/docker-archive/rqlite + skopeo copy docker://kotsadm/rqlite:${RQLITE_TAG} docker-archive:bin/docker-archive/rqlite/${RQLITE_TAG} mkdir -p bin/docker-archive/local-volume-provider skopeo copy docker://replicated/local-volume-provider:${LVP_TAG} docker-archive:bin/docker-archive/local-volume-provider/${LVP_TAG} diff --git a/cmd/imagedeps/README.md b/cmd/imagedeps/README.md index 8962ea024d..732d036ee6 100644 --- a/cmd/imagedeps/README.md +++ b/cmd/imagedeps/README.md @@ -20,27 +20,27 @@ is useful to restrict release tags to a major version, or to filter out garbage | Name | Image URI | Matcher Regexp (Optional) | |------|--------------------|----------| -| Name of the image for example **minio** | Untagged image reference **ghcr.io/dexidp/dex**| An optional regular expression, only matching tags will be included. | +| Name of the image for example **minio** | Untagged image reference **kotsadm/minio**| An optional regular expression, only matching tags will be included. | ### Sample image-spec ```text -minio minio/minio -rqlite rqlite/rqlite -dex ghcr.io/dexidp/dex +minio kotsadm/minio +rqlite kotsadm/rqlite +dex kotsadm/dex ``` The preceding image spec will produce the following environment and Go files. ```shell -MINIO_TAG='RELEASE.2021-09-15T04-54-25Z' -RQLITE_TAG='7.7.0' -DEX_TAG='v2.30.0' +MINIO_TAG='0.20231101.183725' +RQLITE_TAG='7.21.4' +DEX_TAG='2.37.0' ``` ```go package image const ( - Minio = "minio/minio:RELEASE.2021-09-15T04-54-25Z" - Rqlite = "rqlite/rqlite:7.7.0" - Dex = "ghcr.io/dexidp/dex:v2.30.0" + Minio = "kotsadm/minio:0.20231101.183725" + Rqlite = "kotsadm/rqlite:7.21.4" + Dex = "kotsadm/dex:2.37.0" ) ``` diff --git a/cmd/imagedeps/image-spec b/cmd/imagedeps/image-spec index 66a0ae252a..5bdafba0c5 100644 --- a/cmd/imagedeps/image-spec +++ b/cmd/imagedeps/image-spec @@ -1,5 +1,5 @@ -minio minio/minio -rqlite rqlite/rqlite ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ -dex ghcr.io/dexidp/dex +minio kotsadm/minio +rqlite kotsadm/rqlite +dex kotsadm/dex schemahero schemahero/schemahero ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ lvp replicated/local-volume-provider ^v([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ \ No newline at end of file diff --git a/cmd/imagedeps/main.go b/cmd/imagedeps/main.go index 97897a16fb..470116b44a 100644 --- a/cmd/imagedeps/main.go +++ b/cmd/imagedeps/main.go @@ -5,7 +5,6 @@ import ( "bytes" "fmt" "go/format" - "io/ioutil" "log" "os" "regexp" @@ -56,6 +55,9 @@ var ( replacers = []*replacer{ getMakefileReplacer("Makefile"), getMakefileReplacer("migrations/Makefile"), + getApkoFileReplacer("deploy/minio/apko.yaml", "minio"), + getApkoFileReplacer("deploy/rqlite/apko.yaml", "rqlite"), + getApkoFileReplacer("deploy/dex/apko.yaml", "dex"), } ) @@ -146,7 +148,7 @@ func generateOutput(filename, fileTemplate string, refs []*ImageRef, fn template return err } - if err := ioutil.WriteFile(filename, buff, 0644); err != nil { + if err := os.WriteFile(filename, buff, 0644); err != nil { return err } @@ -154,7 +156,7 @@ func generateOutput(filename, fileTemplate string, refs []*ImageRef, fn template } func (r *replacer) replace(refs []*ImageRef) error { - b, err := ioutil.ReadFile(r.path) + b, err := os.ReadFile(r.path) if err != nil { return errors.Wrap(err, "failed to read file") } @@ -168,7 +170,7 @@ func (r *replacer) replace(refs []*ImageRef) error { content = reg.ReplaceAllString(content, r.valueFn(ref)) } - if err := ioutil.WriteFile(r.path, []byte(content), 0644); err != nil { + if err := os.WriteFile(r.path, []byte(content), 0644); err != nil { return errors.Wrap(err, "failed to write file") } @@ -198,7 +200,7 @@ func getMakefileVarName(s string) string { return strings.ToUpper(strings.ReplaceAll(s, "-", "_")) + "_TAG" } -// converts a name from the input string into an a makefile variable name +// converts a name from the input string into an a dockerfile variable name // for example: foo_bar_baz -> FOO_BAR_BAZ func getDockerfileVarName(s string) string { return strings.ToUpper(strings.ReplaceAll(s, "-", "_")) + "_TAG" @@ -227,3 +229,15 @@ func getDockerfileReplacer(path string) *replacer { }, } } + +func getApkoFileReplacer(path string, pkg string) *replacer { + return &replacer{ + path: path, + regexFn: func(ir *ImageRef) string { + return fmt.Sprintf(`- %s~\d+\.\d+\.\d+`, ir.name) + }, + valueFn: func(ir *ImageRef) string { + return ir.GetApkoFileLine(ir.name) + }, + } +} diff --git a/cmd/imagedeps/main_test.go b/cmd/imagedeps/main_test.go index e1ac5eb665..5354fee1a8 100644 --- a/cmd/imagedeps/main_test.go +++ b/cmd/imagedeps/main_test.go @@ -12,16 +12,24 @@ import ( "github.com/stretchr/testify/require" ) -var releaseTags = []string{ - "RELEASE.2022-06-11T19-55-32Z.fips", - "RELEASE.2021-09-09T21-37-06Z.xxx", - "RELEASE.2021-09-09T21-37-05Z", - "RELEASE.2021-09-09T21-37-04Z", -} -var semVerTags = []string{ - "0.12.7", "0.12.6", "0.12.5", - "0.12.4", "0.12.3", "0.12.2", -} +var ( + minioTag = "0.20231101.183725" + rqliteTag = "7.21.4" + dexTag = "2.37.0" + + schemaheroTags = []string{ + "0.13.2", + "0.13.1", + "0.12.7", + "0.12.2", + } + + lvpTags = []string{ + "v0.3.3", + "v0.3.2", + "v0.3.1", + } +) func makeReleases(tags []string) []*github.RepositoryRelease { var releases []*github.RepositoryRelease @@ -46,29 +54,21 @@ func TestFunctional(t *testing.T) { expectError bool }{ { - name: "basic", + name: "minio", fn: getTagFinder( - withGithubReleaseTagFinder( - func(_ string, _ string) ([]*github.RepositoryRelease, error) { - return makeReleases(releaseTags), nil + withWolfiGetTag( + func(_ string) (string, error) { + return minioTag, nil }, ), ), }, { - name: "with-overrides", + name: "schemahero", fn: getTagFinder( withRepoGetTags( func(_ string) ([]string, error) { - return []string{ - "0.13.2", "0.13.1", - "0.12.7", "0.12.2", - }, nil - }, - ), - withGithubReleaseTagFinder( - func(_ string, _ string) ([]*github.RepositoryRelease, error) { - return makeReleases(releaseTags), nil + return schemaheroTags, nil }, ), ), @@ -80,32 +80,19 @@ func TestFunctional(t *testing.T) { { name: "rqlite", fn: getTagFinder( - withRepoGetTags( - func(_ string) ([]string, error) { - return []string{ - "7.7.0", "7.6.1", "7.6.0", - "6.10.2", "6.10.1", "6.8.2", - }, nil - }, - ), - ), - }, - { - name: "filter-github", - fn: getTagFinder( - withGithubReleaseTagFinder( - func(_ string, _ string) ([]*github.RepositoryRelease, error) { - return makeReleases(releaseTags), nil + withWolfiGetTag( + func(_ string) (string, error) { + return rqliteTag, nil }, ), ), }, { - name: "schemahero", + name: "dex", fn: getTagFinder( - withRepoGetTags( - func(_ string) ([]string, error) { - return semVerTags, nil + withWolfiGetTag( + func(_ string) (string, error) { + return dexTag, nil }, ), ), @@ -115,9 +102,7 @@ func TestFunctional(t *testing.T) { fn: getTagFinder( withRepoGetTags( func(_ string) ([]string, error) { - return []string{ - "v0.3.3", - }, nil + return lvpTags, nil }, ), ), diff --git a/cmd/imagedeps/tag-finder.go b/cmd/imagedeps/tag-finder.go index eac106a303..85cad39fb1 100644 --- a/cmd/imagedeps/tag-finder.go +++ b/cmd/imagedeps/tag-finder.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "io" "net/http" "os" "path" @@ -14,6 +15,7 @@ import ( "github.com/google/go-github/v39/github" "github.com/heroku/docker-registry-client/registry" "golang.org/x/oauth2" + "gopkg.in/yaml.v2" ) type ImageRef struct { @@ -42,6 +44,12 @@ func (ir ImageRef) GetDockerfileLine() string { return fmt.Sprintf("ARG %s=%s", getDockerfileVarName(ir.name), ir.tag) } +// GetApkoFileLine generates a line of text intended for use in an Apko file. +func (ir ImageRef) GetApkoFileLine(pkg string) string { + return fmt.Sprintf("- %s~%s", pkg, ir.tag) +} + +type getTagFn func(string) (string, error) type getTagsFn func(string) ([]string, error) type getReleaseFn func(string, string) ([]*github.RepositoryRelease, error) type tagFinderFn func(inputLine string) (*ImageRef, error) @@ -65,6 +73,7 @@ func getFilter(expression string) (filterFn, error) { type configuration struct { repositoryTagsFinder getTagsFn releaseFinder getReleaseFn + wolfiTagFinder getTagFn } // pass to getTagFinder to override the repository tag finder @@ -81,12 +90,20 @@ func withGithubReleaseTagFinder(fn getReleaseFn) func(c *configuration) { } } +// pass to getTagFinder to override the wolfi tag finder +func withWolfiGetTag(fn getTagFn) func(c *configuration) { + return func(c *configuration) { + c.wolfiTagFinder = fn + } +} + // returns a tag finder function that returns information about an image and it's latest tag. func getTagFinder(opts ...func(c *configuration)) tagFinderFn { // set defaults config := configuration{ repositoryTagsFinder: getRegistryTags, releaseFinder: getReleases, + wolfiTagFinder: getLatestTagFromWolfi, } // apply options for _, opt := range opts { @@ -119,19 +136,19 @@ func getTagFinder(opts ...func(c *configuration)) tagFinderFn { switch imageName { case minioReference: - latestReleaseTag, err = getLatestTagFromGithub(config.releaseFinder, "minio", "minio", matcherFn) + latestReleaseTag, err = config.wolfiTagFinder("minio") if err != nil { - return nil, fmt.Errorf("failed to get release tag for minio/minio %w", err) + return nil, fmt.Errorf("failed to get latest minio tag from wolfi %w", err) } case dexReference: - latestReleaseTag, err = getLatestTagFromGithub(config.releaseFinder, "dexidp", "dex", matcherFn) + latestReleaseTag, err = config.wolfiTagFinder("dex") if err != nil { - return nil, fmt.Errorf("failed to get release tag for dexidp/dex %w", err) + return nil, fmt.Errorf("failed to get latest dex tag from wolfi %w", err) } case rqliteReference: - latestReleaseTag, err = getLatestTagFromRegistry("rqlite/rqlite", config.repositoryTagsFinder, matcherFn) + latestReleaseTag, err = config.wolfiTagFinder("rqlite") if err != nil { - return nil, fmt.Errorf("failed to get release tag for %s %w", imageName, err) + return nil, fmt.Errorf("failed to get latest rqlite tag from wolfi %w", err) } case schemaheroReference: latestReleaseTag, err = getLatestTagFromRegistry("schemahero/schemahero", config.repositoryTagsFinder, matcherFn) @@ -274,3 +291,31 @@ func getRegistryTags(untaggedRef string) ([]string, error) { } return tags, nil } + +func getLatestTagFromWolfi(pkg string) (string, error) { + resp, err := http.Get(fmt.Sprintf("https://raw.githubusercontent.com/wolfi-dev/os/main/%s.yaml", pkg)) + if err != nil { + return "", fmt.Errorf("failed to get %s.yaml from wolfi-dev/os: %w", pkg, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read body %w", err) + } + + var yamlData struct { + Package struct { + Version string `yaml:"version"` + } `yaml:"package"` + } + if err := yaml.Unmarshal(body, &yamlData); err != nil { + return "", fmt.Errorf("failed to unmarshal yaml %w", err) + } + + return yamlData.Package.Version, nil +} diff --git a/cmd/imagedeps/testdata/basic/input-spec b/cmd/imagedeps/testdata/basic/input-spec deleted file mode 100644 index 7c0313fd1f..0000000000 --- a/cmd/imagedeps/testdata/basic/input-spec +++ /dev/null @@ -1 +0,0 @@ -minio minio/minio \ No newline at end of file diff --git a/cmd/imagedeps/testdata/filter-github/.image.env b/cmd/imagedeps/testdata/dex/.image.env similarity index 83% rename from cmd/imagedeps/testdata/filter-github/.image.env rename to cmd/imagedeps/testdata/dex/.image.env index 1055c8ca12..3077a28796 100644 --- a/cmd/imagedeps/testdata/filter-github/.image.env +++ b/cmd/imagedeps/testdata/dex/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -MINIO_TAG='RELEASE.2021-09-09T21-37-06Z.xxx' \ No newline at end of file +DEX_TAG='2.37.0' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/filter-github/constants.go b/cmd/imagedeps/testdata/dex/constants.go similarity index 81% rename from cmd/imagedeps/testdata/filter-github/constants.go rename to cmd/imagedeps/testdata/dex/constants.go index 4c28919a57..0a50c62152 100644 --- a/cmd/imagedeps/testdata/filter-github/constants.go +++ b/cmd/imagedeps/testdata/dex/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Minio = "minio/minio:RELEASE.2021-09-09T21-37-06Z.xxx" + Dex = "kotsadm/dex:2.37.0" ) diff --git a/cmd/imagedeps/testdata/dex/input-spec b/cmd/imagedeps/testdata/dex/input-spec new file mode 100644 index 0000000000..a48afc0a87 --- /dev/null +++ b/cmd/imagedeps/testdata/dex/input-spec @@ -0,0 +1 @@ +dex kotsadm/dex \ No newline at end of file diff --git a/cmd/imagedeps/testdata/filter-github/input-spec b/cmd/imagedeps/testdata/filter-github/input-spec deleted file mode 100644 index 80c831f8a5..0000000000 --- a/cmd/imagedeps/testdata/filter-github/input-spec +++ /dev/null @@ -1 +0,0 @@ -minio minio/minio xxx$ \ No newline at end of file diff --git a/cmd/imagedeps/testdata/basic/.image.env b/cmd/imagedeps/testdata/minio/.image.env similarity index 83% rename from cmd/imagedeps/testdata/basic/.image.env rename to cmd/imagedeps/testdata/minio/.image.env index 2999afe7b5..d97660f84f 100644 --- a/cmd/imagedeps/testdata/basic/.image.env +++ b/cmd/imagedeps/testdata/minio/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -MINIO_TAG='RELEASE.2022-06-11T19-55-32Z.fips' \ No newline at end of file +MINIO_TAG='0.20231101.183725' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/basic/constants.go b/cmd/imagedeps/testdata/minio/constants.go similarity index 81% rename from cmd/imagedeps/testdata/basic/constants.go rename to cmd/imagedeps/testdata/minio/constants.go index 240e8c0597..d68ed5eead 100644 --- a/cmd/imagedeps/testdata/basic/constants.go +++ b/cmd/imagedeps/testdata/minio/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Minio = "minio/minio:RELEASE.2022-06-11T19-55-32Z.fips" + Minio = "kotsadm/minio:0.20231101.183725" ) diff --git a/cmd/imagedeps/testdata/minio/input-spec b/cmd/imagedeps/testdata/minio/input-spec new file mode 100644 index 0000000000..128028b8c1 --- /dev/null +++ b/cmd/imagedeps/testdata/minio/input-spec @@ -0,0 +1 @@ +minio kotsadm/minio \ No newline at end of file diff --git a/cmd/imagedeps/testdata/rqlite/.image.env b/cmd/imagedeps/testdata/rqlite/.image.env index 992292b61d..d6f44088a3 100644 --- a/cmd/imagedeps/testdata/rqlite/.image.env +++ b/cmd/imagedeps/testdata/rqlite/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -RQLITE_TAG='7.7.0' \ No newline at end of file +RQLITE_TAG='7.21.4' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/rqlite/constants.go b/cmd/imagedeps/testdata/rqlite/constants.go index ab99db1066..455a24f3a8 100644 --- a/cmd/imagedeps/testdata/rqlite/constants.go +++ b/cmd/imagedeps/testdata/rqlite/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Rqlite = "rqlite/rqlite:7.7.0" + Rqlite = "kotsadm/rqlite:7.21.4" ) diff --git a/cmd/imagedeps/testdata/rqlite/input-spec b/cmd/imagedeps/testdata/rqlite/input-spec index c4772a98f3..fb3664fced 100644 --- a/cmd/imagedeps/testdata/rqlite/input-spec +++ b/cmd/imagedeps/testdata/rqlite/input-spec @@ -1 +1 @@ -rqlite rqlite/rqlite ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ +rqlite kotsadm/rqlite ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ diff --git a/cmd/imagedeps/testdata/schemahero/.image.env b/cmd/imagedeps/testdata/schemahero/.image.env index 931c6000ca..4615bd4e65 100644 --- a/cmd/imagedeps/testdata/schemahero/.image.env +++ b/cmd/imagedeps/testdata/schemahero/.image.env @@ -1,4 +1,4 @@ # Generated file, do not modify. This file is generated from a text file containing a list of images. The # most recent tag is interpolated from the source repository and used to generate a fully qualified image # name. -SCHEMAHERO_TAG='0.12.7' \ No newline at end of file +SCHEMAHERO_TAG='0.13.2' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/schemahero/constants.go b/cmd/imagedeps/testdata/schemahero/constants.go index 50a8eca1f2..3b8bc02ce8 100644 --- a/cmd/imagedeps/testdata/schemahero/constants.go +++ b/cmd/imagedeps/testdata/schemahero/constants.go @@ -5,5 +5,5 @@ package image // image name. const ( - Schemahero = "schemahero/schemahero:0.12.7" + Schemahero = "schemahero/schemahero:0.13.2" ) diff --git a/cmd/imagedeps/testdata/schemahero/input-spec b/cmd/imagedeps/testdata/schemahero/input-spec index 84d47c6bd9..d614df9429 100644 --- a/cmd/imagedeps/testdata/schemahero/input-spec +++ b/cmd/imagedeps/testdata/schemahero/input-spec @@ -1 +1 @@ -schemahero schemahero/schemahero ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ \ No newline at end of file +schemahero schemahero/schemahero ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ diff --git a/cmd/imagedeps/testdata/with-overrides/replacers/expected/test.Dockerfile b/cmd/imagedeps/testdata/schemahero/replacers/expected/test.Dockerfile similarity index 100% rename from cmd/imagedeps/testdata/with-overrides/replacers/expected/test.Dockerfile rename to cmd/imagedeps/testdata/schemahero/replacers/expected/test.Dockerfile diff --git a/cmd/imagedeps/testdata/with-overrides/replacers/expected/test.mk b/cmd/imagedeps/testdata/schemahero/replacers/expected/test.mk similarity index 100% rename from cmd/imagedeps/testdata/with-overrides/replacers/expected/test.mk rename to cmd/imagedeps/testdata/schemahero/replacers/expected/test.mk diff --git a/cmd/imagedeps/testdata/with-overrides/replacers/input/test.Dockerfile b/cmd/imagedeps/testdata/schemahero/replacers/input/test.Dockerfile similarity index 100% rename from cmd/imagedeps/testdata/with-overrides/replacers/input/test.Dockerfile rename to cmd/imagedeps/testdata/schemahero/replacers/input/test.Dockerfile diff --git a/cmd/imagedeps/testdata/with-overrides/replacers/input/test.mk b/cmd/imagedeps/testdata/schemahero/replacers/input/test.mk similarity index 100% rename from cmd/imagedeps/testdata/with-overrides/replacers/input/test.mk rename to cmd/imagedeps/testdata/schemahero/replacers/input/test.mk diff --git a/cmd/imagedeps/testdata/with-overrides/.image.env b/cmd/imagedeps/testdata/with-overrides/.image.env deleted file mode 100644 index f8ab2a50cc..0000000000 --- a/cmd/imagedeps/testdata/with-overrides/.image.env +++ /dev/null @@ -1,5 +0,0 @@ -# Generated file, do not modify. This file is generated from a text file containing a list of images. The -# most recent tag is interpolated from the source repository and used to generate a fully qualified image -# name. -MINIO_TAG='RELEASE.2022-06-11T19-55-32Z.fips' -SCHEMAHERO_TAG='0.13.2' \ No newline at end of file diff --git a/cmd/imagedeps/testdata/with-overrides/constants.go b/cmd/imagedeps/testdata/with-overrides/constants.go deleted file mode 100644 index 99df410f88..0000000000 --- a/cmd/imagedeps/testdata/with-overrides/constants.go +++ /dev/null @@ -1,10 +0,0 @@ -package image - -// Generated file, do not modify. This file is generated from a text file containing a list of images. The -// most recent tag is interpolated from the source repository and used to generate a fully qualified -// image name. - -const ( - Minio = "minio/minio:RELEASE.2022-06-11T19-55-32Z.fips" - Schemahero = "schemahero/schemahero:0.13.2" -) diff --git a/cmd/imagedeps/testdata/with-overrides/input-spec b/cmd/imagedeps/testdata/with-overrides/input-spec deleted file mode 100644 index b2a4505a31..0000000000 --- a/cmd/imagedeps/testdata/with-overrides/input-spec +++ /dev/null @@ -1,2 +0,0 @@ -minio minio/minio -schemahero schemahero/schemahero ^([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)\.([0-9]|[1-9][0-9]*)$ diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl index 276ab70a76..0a711b7e93 100644 --- a/deploy/apko.yaml.tmpl +++ b/deploy/apko.yaml.tmpl @@ -52,7 +52,3 @@ entrypoint: command: /kotsadm cmd: api - -archs: - - x86_64 - - aarch64 diff --git a/deploy/apko_melange_build.md b/deploy/apko_melange_build.md index c67583f632..4d682a598a 100644 --- a/deploy/apko_melange_build.md +++ b/deploy/apko_melange_build.md @@ -30,10 +30,10 @@ melange build melange.yaml --arch=x86_64 > 💡 Only building for your local platform makes builds faster, since it doesn't have to emulate with qemu. > If you're on an arm64 machine (e.g., Apple Silicon), use `--arch=aarch64` here and below. -Then, build the image from the newly built `kots` package, and the other packages needed by the image, using `apko`: +Then, build the image from the newly built `kotsadm` package, and the other packages needed by the image, using `apko`: ```sh -apko publish apko.yaml ttl.sh/kots --arch=x86_64 +apko publish apko.yaml ttl.sh/kotsadm --arch=x86_64 ``` This will print the image to stdout, so you can run it: diff --git a/deploy/dex.Dockerfile b/deploy/dex.Dockerfile deleted file mode 100644 index 7db7f0ff67..0000000000 --- a/deploy/dex.Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -ARG TAG=v2.32.0 -FROM ghcr.io/dexidp/dex:$TAG diff --git a/deploy/dex/apko.yaml b/deploy/dex/apko.yaml new file mode 100644 index 0000000000..5827458515 --- /dev/null +++ b/deploy/dex/apko.yaml @@ -0,0 +1,26 @@ +contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - dex~2.37.0 + - gomplate + - bash + - busybox + - wolfi-baselayout + +accounts: + groups: + - groupname: dex + gid: 1001 + users: + - username: dex + uid: 1001 + gid: 1001 + run-as: dex + +entrypoint: + command: docker-entrypoint + +cmd: dex serve /etc/dex/config.docker.yaml diff --git a/deploy/kurl/kotsadm/template/base/Manifest b/deploy/kurl/kotsadm/template/base/Manifest index 0e57a590cf..f1fb302415 100644 --- a/deploy/kurl/kotsadm/template/base/Manifest +++ b/deploy/kurl/kotsadm/template/base/Manifest @@ -1,7 +1,7 @@ image kotsadm-migrations __KOTSADM_MIGRATIONS_IMAGE__ image kotsadm __KOTSADM_IMAGE__ image kurl-proxy __KURL_PROXY_IMAGE__ -image rqlite rqlite/rqlite:__RQLITE_TAG__ +image rqlite __RQLITE_IMAGE__ image dex __DEX_IMAGE__ asset kots.tar.gz __KOTSADM_BINARY__ diff --git a/deploy/kurl/kotsadm/template/base/rqlite.yaml b/deploy/kurl/kotsadm/template/base/rqlite.yaml index 2a78dd37b1..a71e5d6ff2 100644 --- a/deploy/kurl/kotsadm/template/base/rqlite.yaml +++ b/deploy/kurl/kotsadm/template/base/rqlite.yaml @@ -50,7 +50,7 @@ spec: - kotsadm-rqlite topologyKey: "kubernetes.io/hostname" containers: - - image: rqlite/rqlite:__RQLITE_TAG__ + - image: __RQLITE_IMAGE__ name: rqlite args: - -disco-mode=dns @@ -61,6 +61,8 @@ spec: ports: - name: rqlite containerPort: 4001 + - name: raft + containerPort: 4002 volumeMounts: - name: kotsadm-rqlite mountPath: /rqlite/file diff --git a/deploy/kurl/kotsadm/template/generate.sh b/deploy/kurl/kotsadm/template/generate.sh index 234ea3cd60..a3f82b516a 100755 --- a/deploy/kurl/kotsadm/template/generate.sh +++ b/deploy/kurl/kotsadm/template/generate.sh @@ -28,7 +28,8 @@ function generate() { sed -i -e "s|__KOTSADM_BINARY__|$kotsadm_binary|g" "${dir}/Manifest" # The following environment variables will be exported by the .image.env file - find "$dir" -type f -exec sed -i -e "s|__RQLITE_TAG__|$RQLITE_TAG|g" {} \; + local rqlite_image="$kotsadm_image_registry/$kotsadm_image_namespace/rqlite:$RQLITE_TAG" + find "$dir" -type f -exec sed -i -e "s|__RQLITE_IMAGE__|$rqlite_image|g" {} \; local dex_image="$kotsadm_image_registry/$kotsadm_image_namespace/dex:$DEX_TAG" find "$dir" -type f -exec sed -i -e "s|__DEX_IMAGE__|$dex_image|g" {} \; } diff --git a/deploy/minio/apko.yaml b/deploy/minio/apko.yaml new file mode 100644 index 0000000000..868c6dc88e --- /dev/null +++ b/deploy/minio/apko.yaml @@ -0,0 +1,24 @@ +contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - minio~0.20231101.183725 + - mc + - bash + - busybox + - wolfi-baselayout + +accounts: + groups: + - groupname: minio + gid: 1001 + users: + - username: minio + uid: 1001 + gid: 1001 + run-as: minio + +entrypoint: + command: minio diff --git a/deploy/rqlite/apko.yaml b/deploy/rqlite/apko.yaml new file mode 100644 index 0000000000..5f56ef4c35 --- /dev/null +++ b/deploy/rqlite/apko.yaml @@ -0,0 +1,26 @@ +contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - rqlite~7.21.4 + - rqlite-oci-entrypoint + - bash + - busybox + - wolfi-baselayout + +accounts: + groups: + - groupname: rqlite + gid: 1001 + users: + - username: rqlite + uid: 1001 + gid: 1001 + run-as: rqlite + +entrypoint: + command: docker-entrypoint.sh + +cmd: rqlite diff --git a/kurl_proxy/deploy/apko.yaml.tmpl b/kurl_proxy/deploy/apko.yaml.tmpl index 5dd997f702..e4a32e04f8 100644 --- a/kurl_proxy/deploy/apko.yaml.tmpl +++ b/kurl_proxy/deploy/apko.yaml.tmpl @@ -27,7 +27,3 @@ environment: VERSION: ${GIT_TAG} cmd: /kurl_proxy - -archs: - - x86_64 - - aarch64 diff --git a/migrations/Makefile b/migrations/Makefile index 011acb5480..f28e7bce5e 100644 --- a/migrations/Makefile +++ b/migrations/Makefile @@ -1,19 +1,5 @@ SHELL:=/bin/bash -PROJECT_NAME ?= kotsadm-migrations -RQLITE_TAG ?= 7.21.4 -SCHEMAHERO_TAG ?= 0.16.0 - -.PHONY: schema-alpha -schema-alpha: IMAGE = kotsadm/${PROJECT_NAME}:alpha -schema-alpha: build_schema - -.PHONY: schema-release -schema-release: IMAGE = kotsadm/${PROJECT_NAME}:${GIT_TAG} -schema-release: build_schema - mkdir -p bin/docker-archive/${PROJECT_NAME} - skopeo copy docker-daemon:kotsadm/${PROJECT_NAME}:${GIT_TAG} docker-archive:bin/docker-archive/${PROJECT_NAME}/${GIT_TAG} - mkdir -p bin/docker-archive/rqlite - skopeo copy docker://rqlite/rqlite:${RQLITE_TAG} docker-archive:bin/docker-archive/rqlite/${RQLITE_TAG} +SCHEMAHERO_TAG ?= 0.17.0 build_schema: docker build --pull --build-arg SCHEMAHERO_TAG=${SCHEMAHERO_TAG} -f deploy/Dockerfile -t ${IMAGE} . diff --git a/migrations/deploy/apko.yaml.tmpl b/migrations/deploy/apko.yaml.tmpl new file mode 100644 index 0000000000..0e83a9c503 --- /dev/null +++ b/migrations/deploy/apko.yaml.tmpl @@ -0,0 +1,32 @@ +contents: + repositories: + - https://packages.wolfi.dev/os + - ./packages/ + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + - ./melange.rsa.pub + packages: + - kotsadm-migrations-head # This is expected to be built locally by `melange`. + - bash + - busybox + - curl + - git + - wolfi-baselayout + +accounts: + groups: + - groupname: schemahero + gid: 1001 + users: + - username: schemahero + uid: 1001 + gid: 1001 + run-as: schemahero + +environment: + VERSION: ${GIT_TAG} + +entrypoint: + command: /schemahero + +cmd: apply diff --git a/migrations/deploy/melange.yaml.tmpl b/migrations/deploy/melange.yaml.tmpl new file mode 100644 index 0000000000..88365e8866 --- /dev/null +++ b/migrations/deploy/melange.yaml.tmpl @@ -0,0 +1,32 @@ +package: + name: kotsadm-migrations-head + version: ${GIT_TAG} + epoch: 0 + description: kotsadm-migrations package + copyright: + - license: Apache-2.0 + +environment: + contents: + repositories: + - https://packages.wolfi.dev/os + keyring: + - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + packages: + - ca-certificates-bundle + - busybox + - git + - go + environment: + GOMODCACHE: '/var/cache/melange' + +pipeline: + - runs: | + set -x + export DESTDIR="${{targets.destdir}}" + mkdir -p "${DESTDIR}" + + mv migrations/tables "${DESTDIR}/tables" + + # expected to have already been extracted from the schemahero image + mv migrations/schemahero "${DESTDIR}/schemahero" diff --git a/migrations/kustomize/overlays/dev/rqlite.yaml b/migrations/kustomize/overlays/dev/rqlite.yaml index e23b13defe..4bf62ace05 100644 --- a/migrations/kustomize/overlays/dev/rqlite.yaml +++ b/migrations/kustomize/overlays/dev/rqlite.yaml @@ -59,8 +59,8 @@ spec: spec: containers: - name: rqlite - image: rqlite/rqlite:7.9.2 - imagePullPolicy: IfNotPresent + image: kotsadm/rqlite:alpha + imagePullPolicy: Always args: - -disco-mode=dns - -disco-config={"name":"kotsadm-rqlite-headless"} diff --git a/migrations/kustomize/overlays/okteto/rqlite.yaml b/migrations/kustomize/overlays/okteto/rqlite.yaml index e23b13defe..4bf62ace05 100644 --- a/migrations/kustomize/overlays/okteto/rqlite.yaml +++ b/migrations/kustomize/overlays/okteto/rqlite.yaml @@ -59,8 +59,8 @@ spec: spec: containers: - name: rqlite - image: rqlite/rqlite:7.9.2 - imagePullPolicy: IfNotPresent + image: kotsadm/rqlite:alpha + imagePullPolicy: Always args: - -disco-mode=dns - -disco-config={"name":"kotsadm-rqlite-headless"} diff --git a/pkg/identity/deploy/deploy.go b/pkg/identity/deploy/deploy.go index b7b050a2b8..90d09a795d 100644 --- a/pkg/identity/deploy/deploy.go +++ b/pkg/identity/deploy/deploy.go @@ -462,7 +462,7 @@ func deploymentResource(issuerURL, configChecksum string, options Options) (*app Image: image, ImagePullPolicy: corev1.PullIfNotPresent, Name: "dex", - Command: []string{"/usr/local/bin/dex", "serve", "/etc/dex/cfg/dexConfig.yaml"}, + Command: []string{"dex", "serve", "/etc/dex/cfg/dexConfig.yaml"}, Ports: []corev1.ContainerPort{ {Name: "http", ContainerPort: 5556}, }, diff --git a/pkg/image/constants.go b/pkg/image/constants.go index f1b491f53d..009a983805 100644 --- a/pkg/image/constants.go +++ b/pkg/image/constants.go @@ -5,9 +5,9 @@ package image // image name. const ( - Minio = "minio/minio:RELEASE.2023-11-11T08-14-41Z" - Rqlite = "rqlite/rqlite:7.21.4" - Dex = "ghcr.io/dexidp/dex:v2.37.0" - Schemahero = "schemahero/schemahero:0.16.0" + Minio = "kotsadm/minio:0.20231101.183725" + Rqlite = "kotsadm/rqlite:7.21.4" + Dex = "kotsadm/dex:2.37.0" + Schemahero = "schemahero/schemahero:0.17.0" Lvp = "replicated/local-volume-provider:v0.5.5" ) diff --git a/pkg/kotsadm/minio.go b/pkg/kotsadm/minio.go index 7385b989e9..f6e5a4e928 100644 --- a/pkg/kotsadm/minio.go +++ b/pkg/kotsadm/minio.go @@ -25,7 +25,8 @@ import ( ) var ( - MinioImageTagDateRegexp = regexp.MustCompile(`RELEASE\.(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z)`) + MinioChainguardImageTagRegexp = regexp.MustCompile(`:0\.\d+`) + MinioImageTagDateRegexp = regexp.MustCompile(`RELEASE\.(\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}Z)`) // MigrateToMinioXlBeforeTime is the time that the minio version was released that removed the legacy backend // that we need to migrate from: https://github.com/minio/minio/releases/tag/RELEASE.2022-10-29T06-21-33Z MigrateToMinioXlBeforeTime = time.Date(2022, 10, 29, 6, 21, 33, 0, time.UTC) @@ -127,6 +128,7 @@ func ensureMinioStatefulset(deployOptions types.DeployOptions, clientset kuberne existingMinio.Spec.Template.Spec.Volumes = desiredMinio.Spec.Template.Spec.DeepCopy().Volumes existingMinio.Spec.Template.Spec.Containers[0].Image = desiredMinio.Spec.Template.Spec.Containers[0].Image existingMinio.Spec.Template.Spec.Containers[0].VolumeMounts = desiredMinio.Spec.Template.Spec.Containers[0].DeepCopy().VolumeMounts + existingMinio.Spec.Template.Spec.Containers[0].Command = desiredMinio.Spec.Template.Spec.Containers[0].Command existingMinio.Spec.Template.Spec.InitContainers = desiredMinio.Spec.Template.Spec.DeepCopy().InitContainers _, err = clientset.AppsV1().StatefulSets(deployOptions.Namespace).Update(ctx, existingMinio, metav1.UpdateOptions{}) @@ -393,6 +395,12 @@ func IsMinioXlMigrationNeeded(clientset kubernetes.Interface, namespace string) // imageNeedsMinioXlMigration returns true if the minio image is older than the migrate before time (2022-10-29T06-21-33Z). func imageNeedsMinioXlMigration(minioImage string) (bool, error) { + isCGImage := len(MinioChainguardImageTagRegexp.FindStringSubmatch(minioImage)) > 0 + if isCGImage { + // minio images built with chainguard are all new and don't need to be migrated + return false, nil + } + existingImageTagDateMatch := MinioImageTagDateRegexp.FindStringSubmatch(minioImage) if len(existingImageTagDateMatch) != 2 { return false, errors.New("failed to parse existing image tag date") diff --git a/pkg/kotsadm/minio_test.go b/pkg/kotsadm/minio_test.go index f10fec56b1..6a65f5228b 100644 --- a/pkg/kotsadm/minio_test.go +++ b/pkg/kotsadm/minio_test.go @@ -91,6 +91,13 @@ func Test_IsMinioXlMigrationNeeded(t *testing.T) { wantMinioImage: "minio/minio:RELEASE.2023-02-10T18-48-39Z", wantErr: false, }, + { + name: "should not migrate image built with chainguard", + clientset: fake.NewSimpleClientset(minioStsWithImage("kotsadm/minio:0.20231101.183725")), + wantMigration: false, + wantMinioImage: "kotsadm/minio:0.20231101.183725", + wantErr: false, + }, { name: "should not migrate if no minio", clientset: fake.NewSimpleClientset(), diff --git a/pkg/kotsadm/objects/images.go b/pkg/kotsadm/objects/images.go index 1d097d2611..bfc799b3ce 100644 --- a/pkg/kotsadm/objects/images.go +++ b/pkg/kotsadm/objects/images.go @@ -18,9 +18,9 @@ func GetAdminConsoleImages(deployOptions types.DeployOptions) map[string]string rqliteTag, _ := image.GetTag(image.Rqlite) dexTag, _ := image.GetTag(image.Dex) - minioImage := fmt.Sprintf("minio/minio:%s", minioTag) - rqliteImage := fmt.Sprintf("rqlite/rqlite:%s", rqliteTag) - dexImage := fmt.Sprintf("kotsadm/dex:%s", dexTag) + minioImage := image.Minio + rqliteImage := image.Rqlite + dexImage := image.Dex if s := kotsadmversion.KotsadmPullSecret(deployOptions.Namespace, deployOptions.RegistryConfig); s != nil { minioImage = fmt.Sprintf("%s/minio:%s", kotsadmversion.KotsadmRegistry(deployOptions.RegistryConfig), minioTag) @@ -43,12 +43,11 @@ func GetAdminConsoleImages(deployOptions types.DeployOptions) map[string]string } func GetOriginalAdminConsoleImages(deployOptions types.DeployOptions) map[string]string { - dexTag, _ := image.GetTag(image.Dex) // dex image is special; we host a copy return map[string]string{ "kotsadm-migrations": fmt.Sprintf("kotsadm/kotsadm-migrations:%s", kotsadmversion.KotsadmTag(deployOptions.RegistryConfig)), "kotsadm": fmt.Sprintf("kotsadm/kotsadm:%s", kotsadmversion.KotsadmTag(deployOptions.RegistryConfig)), "minio": image.Minio, "rqlite": image.Rqlite, - "dex": fmt.Sprintf("kotsadm/dex:%s", dexTag), + "dex": image.Dex, } } diff --git a/pkg/kotsadm/objects/minio_objects.go b/pkg/kotsadm/objects/minio_objects.go index 56c12594c7..4ae3acfc76 100644 --- a/pkg/kotsadm/objects/minio_objects.go +++ b/pkg/kotsadm/objects/minio_objects.go @@ -136,7 +136,7 @@ func MinioStatefulset(deployOptions types.DeployOptions, size resource.Quantity) Command: []string{ "/bin/sh", "-ce", - "/usr/bin/docker-entrypoint.sh minio -C /home/minio/.minio/ --quiet server /export", + "minio -C /home/minio/.minio/ --quiet server /export", }, Ports: []corev1.ContainerPort{ { diff --git a/pkg/kotsadm/objects/rqlite_objects.go b/pkg/kotsadm/objects/rqlite_objects.go index 0b750c6e17..b8c56c7a99 100644 --- a/pkg/kotsadm/objects/rqlite_objects.go +++ b/pkg/kotsadm/objects/rqlite_objects.go @@ -128,6 +128,10 @@ func RqliteStatefulset(deployOptions types.DeployOptions, size resource.Quantity Name: "rqlite", ContainerPort: 4001, }, + { + Name: "raft", + ContainerPort: 4002, + }, }, VolumeMounts: volumeMounts, Env: getRqliteEnvs(), diff --git a/pkg/kotsadm/objects/scripts/import-minio-data.sh b/pkg/kotsadm/objects/scripts/import-minio-data.sh index 65acbe11fd..2eb1aef9eb 100644 --- a/pkg/kotsadm/objects/scripts/import-minio-data.sh +++ b/pkg/kotsadm/objects/scripts/import-minio-data.sh @@ -33,7 +33,7 @@ shopt -s dotglob rm -rfv /export/* echo "starting new minio instance" -/bin/sh -ce "/usr/bin/docker-entrypoint.sh minio -C /home/minio/.minio/ server /export" & +/bin/sh -ce "minio -C /home/minio/.minio/ server /export" & MINIO_PID=$! # alias the minio instance diff --git a/pkg/snapshot/filesystem_minio.go b/pkg/snapshot/filesystem_minio.go index cfebe071bd..c188d94bf9 100644 --- a/pkg/snapshot/filesystem_minio.go +++ b/pkg/snapshot/filesystem_minio.go @@ -269,7 +269,12 @@ func fileSystemMinioDeploymentResource(clientset kubernetes.Interface, secretChe if err != nil { return nil, errors.Wrap(err, "failed to get minio image tag") } - minioImage := fmt.Sprintf("minio/minio:%s", minioTag) + + minioImage := fmt.Sprintf("kotsadm/minio:%s", minioTag) + if strings.HasPrefix(minioTag, "RELEASE.") { + minioImage = fmt.Sprintf("minio/minio:%s", minioTag) + } + imagePullSecrets := []corev1.LocalObjectReference{} isKurl, err := kurl.IsKurl(clientset)