From 334881219d6da55696c74de662d644362cc4727c Mon Sep 17 00:00:00 2001
From: Jason Hall <jason@chainguard.dev>
Date: Mon, 26 Jun 2023 17:37:12 -0400
Subject: [PATCH 01/72] build kots with apko+melange in presubmit

Signed-off-by: Jason Hall <jason@chainguard.dev>
---
 .github/workflows/presubmit-image.yaml | 24 +++++++++
 .gitignore                             |  4 ++
 apko.yaml                              | 51 ++++++++++++++++++
 apko_melange_build.md                  | 50 ++++++++++++++++++
 melange.yaml                           | 72 ++++++++++++++++++++++++++
 5 files changed, 201 insertions(+)
 create mode 100644 .github/workflows/presubmit-image.yaml
 create mode 100644 apko.yaml
 create mode 100644 apko_melange_build.md
 create mode 100644 melange.yaml

diff --git a/.github/workflows/presubmit-image.yaml b/.github/workflows/presubmit-image.yaml
new file mode 100644
index 0000000000..7912dee492
--- /dev/null
+++ b/.github/workflows/presubmit-image.yaml
@@ -0,0 +1,24 @@
+on:
+  pull_request:
+
+jobs:
+  presubmit-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '1.20.x'
+      - uses: chainguard-dev/actions/melange-build@main
+        with:
+          config: melange.yaml
+          archs: x86_64
+          sign-with-temporary-key: true
+      - uses: chainguard-images/actions/apko-publish@main
+        with:
+          config: apko.yaml
+          archs: x86_64
+          tag: ttl.sh/kots
+          vcs-url: true
diff --git a/.gitignore b/.gitignore
index 1c73beb175..7850f288c6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,3 +34,7 @@ kotsdata/
 sbom/
 cosign.key
 pkg/tests/pull/cases/*/results
+
+melange.rsa
+melange.rsa.pub
+packages/*
diff --git a/apko.yaml b/apko.yaml
new file mode 100644
index 0000000000..18884fbb10
--- /dev/null
+++ b/apko.yaml
@@ -0,0 +1,51 @@
+contents:
+  repositories:
+    - https://packages.wolfi.dev/os
+    - ./packages/
+  keyring:
+    - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+    - ./melange.rsa.pub
+  packages:
+    - kots-head  # This is expected to be built locally by `melange`.
+
+    - kubectl
+    # https://github.com/wolfi-dev/os/pull/3069
+    #- kubectl-1.27
+    #- kubectl-1.25
+    #- kubectl-1.26
+
+    - bash
+    - busybox
+    - curl
+    - git
+    - helm
+    - kustomize
+    - py3-dateutil
+    - py3-magic
+    - s3cmd
+    - wolfi-baselayout
+
+accounts:
+  groups:
+    - groupname: kotsadm
+      gid: 1001
+  users:
+    - username: kotsadm
+      uid: 1001
+      gid: 1001
+  run-as: kotsadm
+
+environment:
+  VERSION: v1.98.3
+  KOTS_KUBECTL_BIN_DIR: /usr/local/bin
+  KOTS_HELM_BIN_DIR: /usr/local/bin
+  KOTS_KUSTOMIZE_BIN_DIR: /usr/local/bin
+
+entrypoint:
+  command: /kotsadm
+
+cmd: api
+
+archs:
+  - x86_64
+  - aarch64
diff --git a/apko_melange_build.md b/apko_melange_build.md
new file mode 100644
index 0000000000..98390709fe
--- /dev/null
+++ b/apko_melange_build.md
@@ -0,0 +1,50 @@
+# Building KOTS with apko + melange
+
+## What?
+
+- [`melange`](https://github.com/chainguard-dev/melange) is a tool for reproducibly building APK packages from source
+- [`apko`](https://github.com/chainguard-dev/apko) is a tool for reproducibly building container images from APK packages
+
+## Why?
+
+Building with `apko` and `melange` produces smaller, more reproducible images, which can be easier to operate and easier to keep free of vulnerabilities.
+
+## How?
+
+First, build the package from source, using `melange`.
+
+To start, if there isn't already a signing key for the package, we need to generate one:
+
+```sh
+melange keygen
+```
+
+We only need to build for x86_64, which is faster than building for arm64 since it doesn't require qemu.
+
+```sh
+melange build melange.yaml --arch=x86_64
+```
+
+> 💡 Only building for your local platform makes builds faster, since it doesn't have to emulate with qemu.
+> If you're on an arm64 machine (e.g., Apple Silicon), use `--arch=aarch64` here and below.
+
+Then, build the image from the newly built `kots` package, and the other packages needed by the image, using `apko`:
+
+```sh
+apko publish apko.yaml ttl.sh/kots --arch=x86_64
+```
+
+This will print the image to stdout, so you can run it:
+
+```sh
+docker run $(apko publish ...)
+```
+
+### Presubmit GitHub Actions
+
+The above steps are automated in [GitHub Actions](./.github/workflows/presubmit-image.yaml) as a presubmit check for PRs.
+
+## Further Reading
+
+- https://edu.chainguard.dev/open-source/melange/overview/
+- https://edu.chainguard.dev/open-source/apko/overview/
diff --git a/melange.yaml b/melange.yaml
new file mode 100644
index 0000000000..c2709b086d
--- /dev/null
+++ b/melange.yaml
@@ -0,0 +1,72 @@
+package:
+  name: kots-head
+  version: 0.0.1
+  epoch: 0
+  description: Kubernetes Off-The-Shelf (KOTS) Software
+  copyright:
+    - license: Apache-2.0
+
+environment:
+  contents:
+    repositories:
+      - https://packages.wolfi.dev/os
+    keyring:
+      - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+    packages:
+      - ca-certificates-bundle
+      - busybox
+      - git
+      - go
+      - nodejs
+      - yarn
+
+pipeline:
+  - runs: |
+      set -x
+      export DESTDIR="${{targets.destdir}}"
+      mkdir -p "${DESTDIR}"
+
+      # Scripts etc.
+      mv deploy/assets/backup.sh "${DESTDIR}/backup.sh"
+      mv deploy/assets/restore-db.sh "${DESTDIR}/restore-db.sh"
+      mv deploy/assets/restore-s3.sh "${DESTDIR}/restore-s3.sh"
+      mv deploy/assets/restore.sh "${DESTDIR}/restore.sh"
+      mv deploy/assets/migrate-s3.sh "${DESTDIR}/migrate-s3.sh"
+      mv deploy/assets/fs-minio-check.sh "${DESTDIR}/fs-minio-check.sh"
+      mv deploy/assets/fs-minio-reset.sh "${DESTDIR}/fs-minio-reset.sh"
+      mv deploy/assets/fs-minio-keys-sha.sh "${DESTDIR}/fs-minio-keys-sha.sh"
+      mv deploy/assets/s3-bucket-create.sh "${DESTDIR}/s3-bucket-create.sh"
+      mv deploy/assets/s3-bucket-head.sh "${DESTDIR}/s3-bucket-head.sh"
+      mv deploy/assets/kots-upgrade.sh "${DESTDIR}/kots-upgrade.sh"
+      mv deploy/assets/postgres "${DESTDIR}/postgres"
+
+      # kotsadm and kots binaries
+      export VERSION=${{package.version}}
+      export GIT_TAG=${{package.version}}
+
+      # Set environment variables from repository
+      source .image.env
+
+      KOTS_KUSTOMIZE_BIN_DIR=/usr/local/bin
+
+      # TODO: fix pact build error on arm https://github.com/pact-foundation/pact-js-core/issues/264
+      export PACT_SKIP_BINARY_INSTALL=true
+
+      # Configure Yarn
+      yarn install --pure-lockfile --network-concurrency 1
+
+      make -C web deps lint build-kotsadm
+      make vet kots build
+
+      mv bin/kotsadm "${DESTDIR}/kotsadm"
+      mv bin/kots "${DESTDIR}/kots"
+
+      # TODO: this requires manual intervention whenever helm bumps its major version
+      ln -s /usr/bin/helm ${{targets.destdir}}/usr/local/bin/helm
+      ln -s /usr/bin/helm ${{targets.destdir}}/usr/local/bin/helm3
+
+      # TODO: this requires manual intervention whenever kustomize bumps its major version
+      ln -s /usr/bin/kustomize ${{targets.destdir}}/usr/local/bin/kustomize
+      ln -s /usr/bin/kustomize ${{targets.destdir}}/usr/local/bin/kustomize5
+
+      ln -s /usr/bin/kubectl ${{targets.destdir}}/usr/local/bin/kubectl

From c2ccd2a0084ca8df25bf0ff36cf566c37830738e Mon Sep 17 00:00:00 2001
From: Jason Hall <jason@chainguard.dev>
Date: Wed, 28 Jun 2023 18:17:48 -0400
Subject: [PATCH 02/72] add language about non-production-readiness

Signed-off-by: Jason Hall <jason@chainguard.dev>
---
 apko_melange_build.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/apko_melange_build.md b/apko_melange_build.md
index 98390709fe..8500380abb 100644
--- a/apko_melange_build.md
+++ b/apko_melange_build.md
@@ -2,12 +2,14 @@
 
 ## What?
 
+This doc describes a non-production-ready process for building a minimal `kots` image using `melange` and `apko`:
+
 - [`melange`](https://github.com/chainguard-dev/melange) is a tool for reproducibly building APK packages from source
 - [`apko`](https://github.com/chainguard-dev/apko) is a tool for reproducibly building container images from APK packages
 
 ## Why?
 
-Building with `apko` and `melange` produces smaller, more reproducible images, which can be easier to operate and easier to keep free of vulnerabilities.
+Building with `melange` and `apko` produces smaller, more reproducible images, which can be easier to operate and easier to keep free of vulnerabilities.
 
 ## How?
 
@@ -44,6 +46,8 @@ docker run $(apko publish ...)
 
 The above steps are automated in [GitHub Actions](./.github/workflows/presubmit-image.yaml) as a presubmit check for PRs.
 
+The image this workflow produces is only meant for validation, and not meant for production use cases at this time.
+
 ## Further Reading
 
 - https://edu.chainguard.dev/open-source/melange/overview/

From c3c93bc075f0939d2699ff6e8e29b8d33b33d08b Mon Sep 17 00:00:00 2001
From: Jason Hall <jason@chainguard.dev>
Date: Thu, 29 Jun 2023 10:48:09 -0400
Subject: [PATCH 03/72] include all currently supported kubectl versions

Signed-off-by: Jason Hall <jason@chainguard.dev>
---
 apko.yaml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/apko.yaml b/apko.yaml
index 18884fbb10..e87489010b 100644
--- a/apko.yaml
+++ b/apko.yaml
@@ -8,11 +8,12 @@ contents:
   packages:
     - kots-head  # This is expected to be built locally by `melange`.
 
-    - kubectl
-    # https://github.com/wolfi-dev/os/pull/3069
-    #- kubectl-1.27
-    #- kubectl-1.25
-    #- kubectl-1.26
+    # All currently supported kubectl versions.
+    # TODO: this requires manual intervention whenever there are new kubectl releases.
+    - kubectl-1.24
+    - kubectl-1.25
+    - kubectl-1.26
+    - kubectl-1.27
 
     - bash
     - busybox

From e26ba4e8640a886d32deefdb6333b7488c425333 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 25 Aug 2023 21:51:38 +0000
Subject: [PATCH 04/72] initial try

---
 .../actions/build-kotsadm-image/action.yml    | 30 ++++++++++
 .github/workflows/build-test.yaml             | 58 +------------------
 .github/workflows/presubmit-image.yaml        | 24 --------
 melange.yaml                                  |  4 +-
 4 files changed, 35 insertions(+), 81 deletions(-)
 create mode 100644 .github/actions/build-kotsadm-image/action.yml
 delete mode 100644 .github/workflows/presubmit-image.yaml

diff --git a/.github/actions/build-kotsadm-image/action.yml b/.github/actions/build-kotsadm-image/action.yml
new file mode 100644
index 0000000000..d383f96aeb
--- /dev/null
+++ b/.github/actions/build-kotsadm-image/action.yml
@@ -0,0 +1,30 @@
+name: 'Build kotsadm image'
+description: 'Composite action for building kotsadm image for e2e tests'
+inputs:
+  image-name:
+    description: The full image name
+    required: true
+
+runs:
+  using: "composite"
+  permissions:
+    contents: read
+  steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/setup-go@v3
+      with:
+        go-version: '1.20.x'
+
+    - uses: chainguard-dev/actions/melange-build@main
+      with:
+        config: melange.yaml
+        archs: x86_64
+        sign-with-temporary-key: true
+
+    - uses: chainguard-images/actions/apko-publish@main
+      with:
+        config: apko.yaml
+        archs: x86_64
+        tag: ${{ inputs.image-name }}
+        vcs-url: true
diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index feb8a94071..7e0161d80c 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -231,35 +231,6 @@ jobs:
       - name: test
         run: make ci-test
 
-  build-kotsadm:
-    runs-on: ubuntu-20.04
-    needs: [ can-run-ci, build-web, deps-kots, generate-tag ]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: '^1.20.0'
-          cache: true
-
-      - name: Download web artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: web
-          path: ./web/dist
-
-      - name: Build
-        env:
-          GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
-        run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make build
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: kotsadm
-          path: ./bin/kotsadm
 
   build-kots:
     runs-on: ubuntu-20.04
@@ -295,34 +266,11 @@ jobs:
 
   build-push-kotsadm-image:
     runs-on: ubuntu-20.04
-    needs: [ can-run-ci, build-kotsadm, build-kots ]
+    needs: [ can-run-ci ]
     steps:
-      - uses: actions/checkout@v4
+      - uses: ./.github/actions/build-kotsadm-image
         with:
-          ref: ${{github.event.pull_request.head.ref}}
-          repository: ${{github.event.pull_request.head.repo.full_name}}
-
-      - name: Download kots artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: kots
-          path: bin/
-
-      - name: Download kotsadm artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: kotsadm
-          path: bin/
-
-      - run: chmod +x bin/kots bin/kotsadm
-
-      - name: build and push kotsadm for e2e
-        uses: docker/build-push-action@v5
-        with:
-          tags: ttl.sh/automated-${{ github.run_id }}/kotsadm:24h
-          context: ./
-          file: ./deploy/Dockerfile
-          push: true
+          image-name: ttl.sh/automated-${{ github.run_id }}/kotsadm:24h
 
 
   build-kots-helm:
diff --git a/.github/workflows/presubmit-image.yaml b/.github/workflows/presubmit-image.yaml
deleted file mode 100644
index 7912dee492..0000000000
--- a/.github/workflows/presubmit-image.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-on:
-  pull_request:
-
-jobs:
-  presubmit-image:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
-        with:
-          go-version: '1.20.x'
-      - uses: chainguard-dev/actions/melange-build@main
-        with:
-          config: melange.yaml
-          archs: x86_64
-          sign-with-temporary-key: true
-      - uses: chainguard-images/actions/apko-publish@main
-        with:
-          config: apko.yaml
-          archs: x86_64
-          tag: ttl.sh/kots
-          vcs-url: true
diff --git a/melange.yaml b/melange.yaml
index c2709b086d..e1b4cfbd0f 100644
--- a/melange.yaml
+++ b/melange.yaml
@@ -55,8 +55,8 @@ pipeline:
       # Configure Yarn
       yarn install --pure-lockfile --network-concurrency 1
 
-      make -C web deps lint build-kotsadm
-      make vet kots build
+      make -C web deps build-kotsadm
+      make kots build
 
       mv bin/kotsadm "${DESTDIR}/kotsadm"
       mv bin/kots "${DESTDIR}/kots"

From 6d35b8610388852ad8e9f38a46d3791bd69ea784 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 25 Aug 2023 21:53:14 +0000
Subject: [PATCH 05/72] second try

---
 .github/actions/build-kotsadm-image/action.yml | 2 --
 .github/workflows/build-test.yaml              | 1 +
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/actions/build-kotsadm-image/action.yml b/.github/actions/build-kotsadm-image/action.yml
index d383f96aeb..0935afe60e 100644
--- a/.github/actions/build-kotsadm-image/action.yml
+++ b/.github/actions/build-kotsadm-image/action.yml
@@ -10,8 +10,6 @@ runs:
   permissions:
     contents: read
   steps:
-    - uses: actions/checkout@v3
-
     - uses: actions/setup-go@v3
       with:
         go-version: '1.20.x'
diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 7e0161d80c..64591b4dbb 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -268,6 +268,7 @@ jobs:
     runs-on: ubuntu-20.04
     needs: [ can-run-ci ]
     steps:
+      - uses: actions/checkout@v3
       - uses: ./.github/actions/build-kotsadm-image
         with:
           image-name: ttl.sh/automated-${{ github.run_id }}/kotsadm:24h

From a8e05bb1304e5868da206c826beafff1f74fd7a3 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 25 Aug 2023 21:54:13 +0000
Subject: [PATCH 06/72] 3rd try

---
 .github/actions/build-kotsadm-image/action.yml | 6 ------
 .github/workflows/build-test.yaml              | 7 +++++++
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/actions/build-kotsadm-image/action.yml b/.github/actions/build-kotsadm-image/action.yml
index 0935afe60e..7a3d454a97 100644
--- a/.github/actions/build-kotsadm-image/action.yml
+++ b/.github/actions/build-kotsadm-image/action.yml
@@ -7,13 +7,7 @@ inputs:
 
 runs:
   using: "composite"
-  permissions:
-    contents: read
   steps:
-    - uses: actions/setup-go@v3
-      with:
-        go-version: '1.20.x'
-
     - uses: chainguard-dev/actions/melange-build@main
       with:
         config: melange.yaml
diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 64591b4dbb..79be27376a 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -267,8 +267,15 @@ jobs:
   build-push-kotsadm-image:
     runs-on: ubuntu-20.04
     needs: [ can-run-ci ]
+    permissions:
+      contents: read
     steps:
       - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v3
+      with:
+        go-version: '1.20.x'
+
       - uses: ./.github/actions/build-kotsadm-image
         with:
           image-name: ttl.sh/automated-${{ github.run_id }}/kotsadm:24h

From 9a56cb8b9b7b58dac3d9c11d03cbc920591592d1 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 25 Aug 2023 21:54:50 +0000
Subject: [PATCH 07/72] fix syntax error

---
 .github/workflows/build-test.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 79be27376a..ada794ce9c 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -273,8 +273,8 @@ jobs:
       - uses: actions/checkout@v3
 
       - uses: actions/setup-go@v3
-      with:
-        go-version: '1.20.x'
+        with:
+          go-version: '1.20.x'
 
       - uses: ./.github/actions/build-kotsadm-image
         with:

From 446fe1528a70f7d095e30c25f136daacf11957af Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 25 Aug 2023 22:31:01 +0000
Subject: [PATCH 08/72] updates

---
 .../actions/build-kotsadm-image/action.yml    |  4 +-
 .github/workflows/build-test.yaml             | 56 +++++++++----------
 apko.yaml => deploy/apko.yaml                 |  0
 .../apko_melange_build.md                     |  2 +-
 melange.yaml => deploy/melange.yaml           |  0
 5 files changed, 31 insertions(+), 31 deletions(-)
 rename apko.yaml => deploy/apko.yaml (100%)
 rename apko_melange_build.md => deploy/apko_melange_build.md (92%)
 rename melange.yaml => deploy/melange.yaml (100%)

diff --git a/.github/actions/build-kotsadm-image/action.yml b/.github/actions/build-kotsadm-image/action.yml
index 7a3d454a97..2572fcbc68 100644
--- a/.github/actions/build-kotsadm-image/action.yml
+++ b/.github/actions/build-kotsadm-image/action.yml
@@ -10,13 +10,13 @@ runs:
   steps:
     - uses: chainguard-dev/actions/melange-build@main
       with:
-        config: melange.yaml
+        config: deploy/melange.yaml
         archs: x86_64
         sign-with-temporary-key: true
 
     - uses: chainguard-images/actions/apko-publish@main
       with:
-        config: apko.yaml
+        config: deploy/apko.yaml
         archs: x86_64
         tag: ${{ inputs.image-name }}
         vcs-url: true
diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index ada794ce9c..1195094484 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -537,7 +537,7 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           allow-repeats: false
 
-  
+
   cmx-versions:
     runs-on: ubuntu-20.04
     needs: [ enable-tests, can-run-ci ]
@@ -556,7 +556,7 @@ jobs:
 
   validate-existing-online-install-minimal:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -591,7 +591,7 @@ jobs:
 
   validate-smoke-test:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -630,7 +630,7 @@ jobs:
 
   validate-minimal-rbac:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -742,7 +742,7 @@ jobs:
 
   validate-backup-and-restore:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -781,7 +781,7 @@ jobs:
 
   validate-no-required-config:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -821,7 +821,7 @@ jobs:
 
   validate-strict-preflight-checks:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -930,7 +930,7 @@ jobs:
 
   validate-config:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -970,7 +970,7 @@ jobs:
 
   validate-version-history-pagination:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1009,7 +1009,7 @@ jobs:
 
   validate-change-license:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1049,7 +1049,7 @@ jobs:
 
   validate-minimal-rbac-override:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1225,7 +1225,7 @@ jobs:
 
   validate-multi-namespace:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1337,7 +1337,7 @@ jobs:
 
   validate-kots-pull:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1447,7 +1447,7 @@ jobs:
 
   validate-app-version-label:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1605,7 +1605,7 @@ jobs:
 
   validate-helm-install-order:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1704,7 +1704,7 @@ jobs:
 
   validate-no-redeploy-on-restart:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1826,7 +1826,7 @@ jobs:
 
   validate-kubernetes-installer-preflight:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1963,7 +1963,7 @@ jobs:
 
   validate-kots-push-images-anonymous:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -1984,7 +1984,7 @@ jobs:
 
   validate-kots-admin-console-generate-manifests:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -2383,7 +2383,7 @@ jobs:
 
   validate-kots-upgrade:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -2522,7 +2522,7 @@ jobs:
 
   validate-kots-helm-release-secret-migration:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -2709,7 +2709,7 @@ jobs:
 
   validate-multi-app-backup-and-restore:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -2749,7 +2749,7 @@ jobs:
 
   validate-multi-app-install:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -2789,7 +2789,7 @@ jobs:
 
   validate-airgap-smoke-test:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -2830,7 +2830,7 @@ jobs:
 
   validate-remove-app:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -3009,7 +3009,7 @@ jobs:
 
   validate-registry-check:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -3104,7 +3104,7 @@ jobs:
 
   validate-native-helm-v2:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -3330,7 +3330,7 @@ jobs:
 
   validate-deployment-orchestration:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
diff --git a/apko.yaml b/deploy/apko.yaml
similarity index 100%
rename from apko.yaml
rename to deploy/apko.yaml
diff --git a/apko_melange_build.md b/deploy/apko_melange_build.md
similarity index 92%
rename from apko_melange_build.md
rename to deploy/apko_melange_build.md
index 8500380abb..c67583f632 100644
--- a/apko_melange_build.md
+++ b/deploy/apko_melange_build.md
@@ -44,7 +44,7 @@ docker run $(apko publish ...)
 
 ### Presubmit GitHub Actions
 
-The above steps are automated in [GitHub Actions](./.github/workflows/presubmit-image.yaml) as a presubmit check for PRs.
+The above steps are automated in [GitHub Actions](./.github/actions/build-kotsadm-image/action.yml) as a presubmit check for PRs.
 
 The image this workflow produces is only meant for validation, and not meant for production use cases at this time.
 
diff --git a/melange.yaml b/deploy/melange.yaml
similarity index 100%
rename from melange.yaml
rename to deploy/melange.yaml

From de9c3599711d7e17f0f74293a5ad792f73b6a5e2 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 25 Aug 2023 23:11:13 +0000
Subject: [PATCH 09/72] /usr/local/bin

---
 deploy/melange.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/deploy/melange.yaml b/deploy/melange.yaml
index e1b4cfbd0f..a9ba84fa6b 100644
--- a/deploy/melange.yaml
+++ b/deploy/melange.yaml
@@ -62,11 +62,11 @@ pipeline:
       mv bin/kots "${DESTDIR}/kots"
 
       # TODO: this requires manual intervention whenever helm bumps its major version
-      ln -s /usr/bin/helm ${{targets.destdir}}/usr/local/bin/helm
-      ln -s /usr/bin/helm ${{targets.destdir}}/usr/local/bin/helm3
+      ln -s /usr/local/bin/helm ${{targets.destdir}}/usr/local/bin/helm
+      ln -s /usr/local/bin/helm ${{targets.destdir}}/usr/local/bin/helm3
 
       # TODO: this requires manual intervention whenever kustomize bumps its major version
-      ln -s /usr/bin/kustomize ${{targets.destdir}}/usr/local/bin/kustomize
-      ln -s /usr/bin/kustomize ${{targets.destdir}}/usr/local/bin/kustomize5
+      ln -s /usr/local/bin/kustomize ${{targets.destdir}}/usr/local/bin/kustomize
+      ln -s /usr/local/bin/kustomize ${{targets.destdir}}/usr/local/bin/kustomize5
 
-      ln -s /usr/bin/kubectl ${{targets.destdir}}/usr/local/bin/kubectl
+      ln -s /usr/local/bin/kubectl ${{targets.destdir}}/usr/local/bin/kubectl

From f8f6f3ca8d2869896a4b50a34e1e4f0630768ace Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 25 Aug 2023 23:43:26 +0000
Subject: [PATCH 10/72] fix symlinks

---
 deploy/apko.yaml    |  2 +-
 deploy/melange.yaml | 19 ++++++++++++-------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/deploy/apko.yaml b/deploy/apko.yaml
index e87489010b..7de25da28b 100644
--- a/deploy/apko.yaml
+++ b/deploy/apko.yaml
@@ -6,7 +6,7 @@ contents:
     - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
     - ./melange.rsa.pub
   packages:
-    - kots-head  # This is expected to be built locally by `melange`.
+    - kotsadm-head  # This is expected to be built locally by `melange`.
 
     # All currently supported kubectl versions.
     # TODO: this requires manual intervention whenever there are new kubectl releases.
diff --git a/deploy/melange.yaml b/deploy/melange.yaml
index a9ba84fa6b..6d307bac6c 100644
--- a/deploy/melange.yaml
+++ b/deploy/melange.yaml
@@ -1,8 +1,8 @@
 package:
-  name: kots-head
+  name: kotsadm-head
   version: 0.0.1
   epoch: 0
-  description: Kubernetes Off-The-Shelf (KOTS) Software
+  description: Kotsadm package
   copyright:
     - license: Apache-2.0
 
@@ -62,11 +62,16 @@ pipeline:
       mv bin/kots "${DESTDIR}/kots"
 
       # TODO: this requires manual intervention whenever helm bumps its major version
-      ln -s /usr/local/bin/helm ${{targets.destdir}}/usr/local/bin/helm
-      ln -s /usr/local/bin/helm ${{targets.destdir}}/usr/local/bin/helm3
+      ln -s /usr/bin/helm ${{targets.destdir}}/usr/local/bin/helm
+      ln -s /usr/bin/helm ${{targets.destdir}}/usr/local/bin/helm3
 
       # TODO: this requires manual intervention whenever kustomize bumps its major version
-      ln -s /usr/local/bin/kustomize ${{targets.destdir}}/usr/local/bin/kustomize
-      ln -s /usr/local/bin/kustomize ${{targets.destdir}}/usr/local/bin/kustomize5
+      ln -s /usr/bin/kustomize ${{targets.destdir}}/usr/local/bin/kustomize
+      ln -s /usr/bin/kustomize ${{targets.destdir}}/usr/local/bin/kustomize5
 
-      ln -s /usr/local/bin/kubectl ${{targets.destdir}}/usr/local/bin/kubectl
+      # TODO: this requires manual intervention whenever kubectl bumps its major version
+      ln -s /usr/bin/kubectl-1.24 ${{targets.destdir}}/usr/local/bin/kubectl-v1.24
+      ln -s /usr/bin/kubectl-1.25 ${{targets.destdir}}/usr/local/bin/kubectl-v1.25
+      ln -s /usr/bin/kubectl-1.26 ${{targets.destdir}}/usr/local/bin/kubectl-v1.26
+      ln -s /usr/bin/kubectl-1.27 ${{targets.destdir}}/usr/local/bin/kubectl-v1.27
+      ln -s /usr/bin/kubectl-1.27 ${{targets.destdir}}/usr/local/bin/kubectl

From 45088525c5db66df3a10911c893f40662b6b4e37 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Sat, 26 Aug 2023 00:23:13 +0000
Subject: [PATCH 11/72] propagate git tag to melange and apko + always cleanup
 c11y clusters

---
 .../actions/build-kotsadm-image/action.yml    | 22 -------------------
 .github/workflows/build-test.yaml             | 20 +++++++++++++++--
 deploy/{apko.yaml => apko.yaml.tmpl}          |  2 +-
 deploy/{melange.yaml => melange.yaml.tmpl}    |  2 +-
 4 files changed, 20 insertions(+), 26 deletions(-)
 delete mode 100644 .github/actions/build-kotsadm-image/action.yml
 rename deploy/{apko.yaml => apko.yaml.tmpl} (97%)
 rename deploy/{melange.yaml => melange.yaml.tmpl} (99%)

diff --git a/.github/actions/build-kotsadm-image/action.yml b/.github/actions/build-kotsadm-image/action.yml
deleted file mode 100644
index 2572fcbc68..0000000000
--- a/.github/actions/build-kotsadm-image/action.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-name: 'Build kotsadm image'
-description: 'Composite action for building kotsadm image for e2e tests'
-inputs:
-  image-name:
-    description: The full image name
-    required: true
-
-runs:
-  using: "composite"
-  steps:
-    - uses: chainguard-dev/actions/melange-build@main
-      with:
-        config: deploy/melange.yaml
-        archs: x86_64
-        sign-with-temporary-key: true
-
-    - uses: chainguard-images/actions/apko-publish@main
-      with:
-        config: deploy/apko.yaml
-        archs: x86_64
-        tag: ${{ inputs.image-name }}
-        vcs-url: true
diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 1195094484..0a2129a6ee 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -276,9 +276,25 @@ jobs:
         with:
           go-version: '1.20.x'
 
-      - uses: ./.github/actions/build-kotsadm-image
+      - name: template melange and apko configs
+        env:
+          GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
+        run: |
+          envsubst < deploy/melange.yaml.tmpl > deploy/melange.yaml
+          envsubst < deploy/apko.yaml.tmpl > deploy/apko.yaml
+
+      - uses: chainguard-dev/actions/melange-build@main
+        with:
+          config: deploy/melange.yaml
+          archs: x86_64
+          sign-with-temporary-key: true
+
+      - uses: chainguard-images/actions/apko-publish@main
         with:
-          image-name: ttl.sh/automated-${{ github.run_id }}/kotsadm:24h
+          config: deploy/apko.yaml
+          archs: x86_64
+          tag: ttl.sh/automated-${{ github.run_id }}/kotsadm:24h
+          vcs-url: true
 
 
   build-kots-helm:
diff --git a/deploy/apko.yaml b/deploy/apko.yaml.tmpl
similarity index 97%
rename from deploy/apko.yaml
rename to deploy/apko.yaml.tmpl
index 7de25da28b..241a90075d 100644
--- a/deploy/apko.yaml
+++ b/deploy/apko.yaml.tmpl
@@ -37,7 +37,7 @@ accounts:
   run-as: kotsadm
 
 environment:
-  VERSION: v1.98.3
+  VERSION: ${GIT_TAG}
   KOTS_KUBECTL_BIN_DIR: /usr/local/bin
   KOTS_HELM_BIN_DIR: /usr/local/bin
   KOTS_KUSTOMIZE_BIN_DIR: /usr/local/bin
diff --git a/deploy/melange.yaml b/deploy/melange.yaml.tmpl
similarity index 99%
rename from deploy/melange.yaml
rename to deploy/melange.yaml.tmpl
index 6d307bac6c..80ebc62ce6 100644
--- a/deploy/melange.yaml
+++ b/deploy/melange.yaml.tmpl
@@ -1,6 +1,6 @@
 package:
   name: kotsadm-head
-  version: 0.0.1
+  version: ${GIT_TAG}
   epoch: 0
   description: Kotsadm package
   copyright:

From 9c0338c33a3fad67780628b8c9878fd5154779ae Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 28 Aug 2023 16:39:55 +0000
Subject: [PATCH 12/72] fix templating

---
 .github/workflows/build-test.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 0a2129a6ee..56470592f7 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -277,9 +277,8 @@ jobs:
           go-version: '1.20.x'
 
       - name: template melange and apko configs
-        env:
-          GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
         run: |
+          export GIT_TAG=${{ needs.generate-tag.outputs.tag }}
           envsubst < deploy/melange.yaml.tmpl > deploy/melange.yaml
           envsubst < deploy/apko.yaml.tmpl > deploy/apko.yaml
 

From 357967dc7b9e8b82cc71de37f2935071b0134403 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 28 Aug 2023 16:41:23 +0000
Subject: [PATCH 13/72] more fixes

---
 .github/workflows/build-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 56470592f7..1684a2556e 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -266,7 +266,7 @@ jobs:
 
   build-push-kotsadm-image:
     runs-on: ubuntu-20.04
-    needs: [ can-run-ci ]
+    needs: [ can-run-ci, generate-tag ]
     permissions:
       contents: read
     steps:

From e6e2b887d2420a54649f9fb01d024c346be70efe Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 28 Aug 2023 17:02:47 +0000
Subject: [PATCH 14/72] fix melange target dir env var

---
 deploy/melange.yaml.tmpl | 31 +++++++++++++++----------------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index 80ebc62ce6..a1b9d8e580 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -23,22 +23,21 @@ environment:
 pipeline:
   - runs: |
       set -x
-      export DESTDIR="${{targets.destdir}}"
-      mkdir -p "${DESTDIR}"
+      mkdir -p "${{targets.destdir}}"
 
       # Scripts etc.
-      mv deploy/assets/backup.sh "${DESTDIR}/backup.sh"
-      mv deploy/assets/restore-db.sh "${DESTDIR}/restore-db.sh"
-      mv deploy/assets/restore-s3.sh "${DESTDIR}/restore-s3.sh"
-      mv deploy/assets/restore.sh "${DESTDIR}/restore.sh"
-      mv deploy/assets/migrate-s3.sh "${DESTDIR}/migrate-s3.sh"
-      mv deploy/assets/fs-minio-check.sh "${DESTDIR}/fs-minio-check.sh"
-      mv deploy/assets/fs-minio-reset.sh "${DESTDIR}/fs-minio-reset.sh"
-      mv deploy/assets/fs-minio-keys-sha.sh "${DESTDIR}/fs-minio-keys-sha.sh"
-      mv deploy/assets/s3-bucket-create.sh "${DESTDIR}/s3-bucket-create.sh"
-      mv deploy/assets/s3-bucket-head.sh "${DESTDIR}/s3-bucket-head.sh"
-      mv deploy/assets/kots-upgrade.sh "${DESTDIR}/kots-upgrade.sh"
-      mv deploy/assets/postgres "${DESTDIR}/postgres"
+      mv deploy/assets/backup.sh "${{targets.destdir}}/backup.sh"
+      mv deploy/assets/restore-db.sh "${{targets.destdir}}/restore-db.sh"
+      mv deploy/assets/restore-s3.sh "${{targets.destdir}}/restore-s3.sh"
+      mv deploy/assets/restore.sh "${{targets.destdir}}/restore.sh"
+      mv deploy/assets/migrate-s3.sh "${{targets.destdir}}/migrate-s3.sh"
+      mv deploy/assets/fs-minio-check.sh "${{targets.destdir}}/fs-minio-check.sh"
+      mv deploy/assets/fs-minio-reset.sh "${{targets.destdir}}/fs-minio-reset.sh"
+      mv deploy/assets/fs-minio-keys-sha.sh "${{targets.destdir}}/fs-minio-keys-sha.sh"
+      mv deploy/assets/s3-bucket-create.sh "${{targets.destdir}}/s3-bucket-create.sh"
+      mv deploy/assets/s3-bucket-head.sh "${{targets.destdir}}/s3-bucket-head.sh"
+      mv deploy/assets/kots-upgrade.sh "${{targets.destdir}}/kots-upgrade.sh"
+      mv deploy/assets/postgres "${{targets.destdir}}/postgres"
 
       # kotsadm and kots binaries
       export VERSION=${{package.version}}
@@ -58,8 +57,8 @@ pipeline:
       make -C web deps build-kotsadm
       make kots build
 
-      mv bin/kotsadm "${DESTDIR}/kotsadm"
-      mv bin/kots "${DESTDIR}/kots"
+      mv bin/kotsadm "${{targets.destdir}}/kotsadm"
+      mv bin/kots "${{targets.destdir}}/kots"
 
       # TODO: this requires manual intervention whenever helm bumps its major version
       ln -s /usr/bin/helm ${{targets.destdir}}/usr/local/bin/helm

From dd095d66207cda1e0fde94e979d53f00ef61b8da Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 28 Aug 2023 17:49:05 +0000
Subject: [PATCH 15/72] one more try

---
 .github/workflows/build-test.yaml |  4 +--
 deploy/melange.yaml.tmpl          | 51 ++++++++++++++++---------------
 2 files changed, 29 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 1684a2556e..f2c80aaff4 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -279,8 +279,8 @@ jobs:
       - name: template melange and apko configs
         run: |
           export GIT_TAG=${{ needs.generate-tag.outputs.tag }}
-          envsubst < deploy/melange.yaml.tmpl > deploy/melange.yaml
-          envsubst < deploy/apko.yaml.tmpl > deploy/apko.yaml
+          envsubst '${GIT_TAG}' < deploy/melange.yaml.tmpl > deploy/melange.yaml
+          envsubst '${GIT_TAG}' < deploy/apko.yaml.tmpl > deploy/apko.yaml
 
       - uses: chainguard-dev/actions/melange-build@main
         with:
diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index a1b9d8e580..e72b4f3d14 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -23,21 +23,22 @@ environment:
 pipeline:
   - runs: |
       set -x
-      mkdir -p "${{targets.destdir}}"
+      export DESTDIR="${{targets.destdir}}"
+      mkdir -p "${DESTDIR}"
 
       # Scripts etc.
-      mv deploy/assets/backup.sh "${{targets.destdir}}/backup.sh"
-      mv deploy/assets/restore-db.sh "${{targets.destdir}}/restore-db.sh"
-      mv deploy/assets/restore-s3.sh "${{targets.destdir}}/restore-s3.sh"
-      mv deploy/assets/restore.sh "${{targets.destdir}}/restore.sh"
-      mv deploy/assets/migrate-s3.sh "${{targets.destdir}}/migrate-s3.sh"
-      mv deploy/assets/fs-minio-check.sh "${{targets.destdir}}/fs-minio-check.sh"
-      mv deploy/assets/fs-minio-reset.sh "${{targets.destdir}}/fs-minio-reset.sh"
-      mv deploy/assets/fs-minio-keys-sha.sh "${{targets.destdir}}/fs-minio-keys-sha.sh"
-      mv deploy/assets/s3-bucket-create.sh "${{targets.destdir}}/s3-bucket-create.sh"
-      mv deploy/assets/s3-bucket-head.sh "${{targets.destdir}}/s3-bucket-head.sh"
-      mv deploy/assets/kots-upgrade.sh "${{targets.destdir}}/kots-upgrade.sh"
-      mv deploy/assets/postgres "${{targets.destdir}}/postgres"
+      mv deploy/assets/backup.sh "${DESTDIR}/backup.sh"
+      mv deploy/assets/restore-db.sh "${DESTDIR}/restore-db.sh"
+      mv deploy/assets/restore-s3.sh "${DESTDIR}/restore-s3.sh"
+      mv deploy/assets/restore.sh "${DESTDIR}/restore.sh"
+      mv deploy/assets/migrate-s3.sh "${DESTDIR}/migrate-s3.sh"
+      mv deploy/assets/fs-minio-check.sh "${DESTDIR}/fs-minio-check.sh"
+      mv deploy/assets/fs-minio-reset.sh "${DESTDIR}/fs-minio-reset.sh"
+      mv deploy/assets/fs-minio-keys-sha.sh "${DESTDIR}/fs-minio-keys-sha.sh"
+      mv deploy/assets/s3-bucket-create.sh "${DESTDIR}/s3-bucket-create.sh"
+      mv deploy/assets/s3-bucket-head.sh "${DESTDIR}/s3-bucket-head.sh"
+      mv deploy/assets/kots-upgrade.sh "${DESTDIR}/kots-upgrade.sh"
+      mv deploy/assets/postgres "${DESTDIR}/postgres"
 
       # kotsadm and kots binaries
       export VERSION=${{package.version}}
@@ -46,7 +47,9 @@ pipeline:
       # Set environment variables from repository
       source .image.env
 
+      KOTS_KUBECTL_BIN_DIR=/usr/local/bin
       KOTS_KUSTOMIZE_BIN_DIR=/usr/local/bin
+      KOTS_HELM_BIN_DIR=/usr/local/bin
 
       # TODO: fix pact build error on arm https://github.com/pact-foundation/pact-js-core/issues/264
       export PACT_SKIP_BINARY_INSTALL=true
@@ -57,20 +60,20 @@ pipeline:
       make -C web deps build-kotsadm
       make kots build
 
-      mv bin/kotsadm "${{targets.destdir}}/kotsadm"
-      mv bin/kots "${{targets.destdir}}/kots"
+      mv bin/kotsadm "${DESTDIR}/kotsadm"
+      mv bin/kots "${DESTDIR}/kots"
 
       # TODO: this requires manual intervention whenever helm bumps its major version
-      ln -s /usr/bin/helm ${{targets.destdir}}/usr/local/bin/helm
-      ln -s /usr/bin/helm ${{targets.destdir}}/usr/local/bin/helm3
+      ln -s /usr/bin/helm ${DESTDIR}/usr/local/bin/helm
+      ln -s /usr/bin/helm ${DESTDIR}/usr/local/bin/helm3
 
       # TODO: this requires manual intervention whenever kustomize bumps its major version
-      ln -s /usr/bin/kustomize ${{targets.destdir}}/usr/local/bin/kustomize
-      ln -s /usr/bin/kustomize ${{targets.destdir}}/usr/local/bin/kustomize5
+      ln -s /usr/bin/kustomize ${DESTDIR}/usr/local/bin/kustomize
+      ln -s /usr/bin/kustomize ${DESTDIR}/usr/local/bin/kustomize5
 
       # TODO: this requires manual intervention whenever kubectl bumps its major version
-      ln -s /usr/bin/kubectl-1.24 ${{targets.destdir}}/usr/local/bin/kubectl-v1.24
-      ln -s /usr/bin/kubectl-1.25 ${{targets.destdir}}/usr/local/bin/kubectl-v1.25
-      ln -s /usr/bin/kubectl-1.26 ${{targets.destdir}}/usr/local/bin/kubectl-v1.26
-      ln -s /usr/bin/kubectl-1.27 ${{targets.destdir}}/usr/local/bin/kubectl-v1.27
-      ln -s /usr/bin/kubectl-1.27 ${{targets.destdir}}/usr/local/bin/kubectl
+      ln -s /usr/bin/kubectl-1.24 ${DESTDIR}/usr/local/bin/kubectl-v1.24
+      ln -s /usr/bin/kubectl-1.25 ${DESTDIR}/usr/local/bin/kubectl-v1.25
+      ln -s /usr/bin/kubectl-1.26 ${DESTDIR}/usr/local/bin/kubectl-v1.26
+      ln -s /usr/bin/kubectl-1.27 ${DESTDIR}/usr/local/bin/kubectl-v1.27
+      ln -s /usr/bin/kubectl-1.27 ${DESTDIR}/usr/local/bin/kubectl

From 34f2f3adc5e734fc32e71903888152f175cd1498 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 6 Oct 2023 21:17:19 +0000
Subject: [PATCH 16/72] no-op, trigger tests again


From 3fbdee1a064bb358031a8274334d48396df3c652 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Fri, 6 Oct 2023 21:18:23 +0000
Subject: [PATCH 17/72] fix lint isses

---
 .github/workflows/build-test.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index f2c80aaff4..7e7fd0d1fc 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -279,8 +279,8 @@ jobs:
       - name: template melange and apko configs
         run: |
           export GIT_TAG=${{ needs.generate-tag.outputs.tag }}
-          envsubst '${GIT_TAG}' < deploy/melange.yaml.tmpl > deploy/melange.yaml
-          envsubst '${GIT_TAG}' < deploy/apko.yaml.tmpl > deploy/apko.yaml
+          envsubst "${GIT_TAG}" < deploy/melange.yaml.tmpl > deploy/melange.yaml
+          envsubst "${GIT_TAG}" < deploy/apko.yaml.tmpl > deploy/apko.yaml
 
       - uses: chainguard-dev/actions/melange-build@main
         with:

From 81d806354d095dfa1a84058842a5370d055a0ece Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 15:18:01 +0000
Subject: [PATCH 18/72] use melange caching

---
 .github/workflows/build-test.yaml | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 7e7fd0d1fc..b5f0093605 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -274,7 +274,8 @@ jobs:
 
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.20.x'
+          go-version: '^1.20.0'
+          cache: true
 
       - name: template melange and apko configs
         run: |
@@ -282,11 +283,23 @@ jobs:
           envsubst "${GIT_TAG}" < deploy/melange.yaml.tmpl > deploy/melange.yaml
           envsubst "${GIT_TAG}" < deploy/apko.yaml.tmpl > deploy/apko.yaml
 
-      - uses: chainguard-dev/actions/melange-build@main
-        with:
-          config: deploy/melange.yaml
-          archs: x86_64
-          sign-with-temporary-key: true
+      # use chainguard-dev/actions/melange-build@main action below instead when caching support is added
+      - name: set up melange
+        uses: chainguard-dev/actions/setup-melange@main
+
+      - name: build with melange
+        run: |
+          melange build \
+            --config deploy/melange.yaml \
+            --archs x86_64 \
+            --signing-key ${{ github.workspace }}/melange.rsa \
+            --cache-dir "$(go env GOMODCACHE)"
+
+      # - uses: chainguard-dev/actions/melange-build@main
+      #   with:
+      #     config: deploy/melange.yaml
+      #     archs: x86_64
+      #     sign-with-temporary-key: true
 
       - uses: chainguard-images/actions/apko-publish@main
         with:

From 48a72e93c6ae492541ee957f4fd24e9edf67b5c9 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 15:48:42 +0000
Subject: [PATCH 19/72] generate signing key

---
 .github/workflows/build-test.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index b5f0093605..73caed2c14 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -284,13 +284,14 @@ jobs:
           envsubst "${GIT_TAG}" < deploy/apko.yaml.tmpl > deploy/apko.yaml
 
       # use chainguard-dev/actions/melange-build@main action below instead when caching support is added
-      - name: set up melange
-        uses: chainguard-dev/actions/setup-melange@main
+      - uses: chainguard-dev/actions/setup-melange@main
+      - uses: chainguard-dev/actions/melange-keygen@main
+        with:
+          signing-key-path: ${{ github.workspace }}/melange.rsa
 
       - name: build with melange
         run: |
-          melange build \
-            --config deploy/melange.yaml \
+          melange build deploy/melange.yaml \
             --archs x86_64 \
             --signing-key ${{ github.workspace }}/melange.rsa \
             --cache-dir "$(go env GOMODCACHE)"

From 33df7c10d08ea1bab4d92a74866884eeded9af92 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 15:56:55 +0000
Subject: [PATCH 20/72] fix flag name

---
 .github/workflows/build-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 73caed2c14..3da4a5e40a 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -292,7 +292,7 @@ jobs:
       - name: build with melange
         run: |
           melange build deploy/melange.yaml \
-            --archs x86_64 \
+            --arch x86_64 \
             --signing-key ${{ github.workspace }}/melange.rsa \
             --cache-dir "$(go env GOMODCACHE)"
 

From 2112fa40bda19b8fa6bf35241e9aeb7beab4abe8 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 16:06:41 +0000
Subject: [PATCH 21/72] test without cache again

---
 .github/workflows/build-test.yaml | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 3da4a5e40a..dcedff0f80 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -284,23 +284,23 @@ jobs:
           envsubst "${GIT_TAG}" < deploy/apko.yaml.tmpl > deploy/apko.yaml
 
       # use chainguard-dev/actions/melange-build@main action below instead when caching support is added
-      - uses: chainguard-dev/actions/setup-melange@main
-      - uses: chainguard-dev/actions/melange-keygen@main
-        with:
-          signing-key-path: ${{ github.workspace }}/melange.rsa
+      # - uses: chainguard-dev/actions/setup-melange@main
+      # - uses: chainguard-dev/actions/melange-keygen@main
+      #   with:
+      #     signing-key-path: ${{ github.workspace }}/melange.rsa
 
-      - name: build with melange
-        run: |
-          melange build deploy/melange.yaml \
-            --arch x86_64 \
-            --signing-key ${{ github.workspace }}/melange.rsa \
-            --cache-dir "$(go env GOMODCACHE)"
+      # - name: build with melange
+      #   run: |
+      #     melange build deploy/melange.yaml \
+      #       --arch x86_64 \
+      #       --signing-key ${{ github.workspace }}/melange.rsa \
+      #       --cache-dir "$(go env GOMODCACHE)"
 
-      # - uses: chainguard-dev/actions/melange-build@main
-      #   with:
-      #     config: deploy/melange.yaml
-      #     archs: x86_64
-      #     sign-with-temporary-key: true
+      - uses: chainguard-dev/actions/melange-build@main
+        with:
+          config: deploy/melange.yaml
+          archs: x86_64
+          sign-with-temporary-key: true
 
       - uses: chainguard-images/actions/apko-publish@main
         with:

From 9deff023911082103401619823dafd5ff250b6db Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 16:25:53 +0000
Subject: [PATCH 22/72] test caching support

---
 .github/workflows/build-test.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index dcedff0f80..2cdc95a699 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -296,7 +296,8 @@ jobs:
       #       --signing-key ${{ github.workspace }}/melange.rsa \
       #       --cache-dir "$(go env GOMODCACHE)"
 
-      - uses: chainguard-dev/actions/melange-build@main
+      # - uses: chainguard-dev/actions/melange-build@main
+      - uses: sgalsaleh/chainguard-dev-actions@4824ef7293e21e9d66749bd24cfd2a11f9d05dfa
         with:
           config: deploy/melange.yaml
           archs: x86_64

From d3775c4888260742ad5cac5f8af467cd1cbdb797 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 16:29:13 +0000
Subject: [PATCH 23/72] fix action

---
 .github/workflows/build-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 2cdc95a699..d0d0e98801 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -297,7 +297,7 @@ jobs:
       #       --cache-dir "$(go env GOMODCACHE)"
 
       # - uses: chainguard-dev/actions/melange-build@main
-      - uses: sgalsaleh/chainguard-dev-actions@4824ef7293e21e9d66749bd24cfd2a11f9d05dfa
+      - uses: sgalsaleh/chainguard-dev-actions/melange-build@4824ef7293e21e9d66749bd24cfd2a11f9d05dfa
         with:
           config: deploy/melange.yaml
           archs: x86_64

From 33f3b750e3720db3b78425b3fc6d207f6cdaaa56 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 16:38:06 +0000
Subject: [PATCH 24/72] support go mod cache

---
 .github/workflows/build-test.yaml | 4 ++++
 deploy/melange.yaml.tmpl          | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index d0d0e98801..37a36e9370 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -296,12 +296,16 @@ jobs:
       #       --signing-key ${{ github.workspace }}/melange.rsa \
       #       --cache-dir "$(go env GOMODCACHE)"
 
+      - id: cache-dir
+        run: echo "cache_dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
+
       # - uses: chainguard-dev/actions/melange-build@main
       - uses: sgalsaleh/chainguard-dev-actions/melange-build@4824ef7293e21e9d66749bd24cfd2a11f9d05dfa
         with:
           config: deploy/melange.yaml
           archs: x86_64
           sign-with-temporary-key: true
+          cache-dir: ${{ steps.cache-dir.outputs.cache_dir }}
 
       - uses: chainguard-images/actions/apko-publish@main
         with:
diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index e72b4f3d14..d2a2ad7523 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -19,6 +19,8 @@ environment:
       - go
       - nodejs
       - yarn
+  environment:
+    GOMODCACHE: '/var/cache/melange'
 
 pipeline:
   - runs: |

From 434e537ef331c1a9151d0134e2b0a19a47bf5a44 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 17:39:41 +0000
Subject: [PATCH 25/72] one more test

---
 .github/workflows/build-test.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 37a36e9370..864b597831 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -272,10 +272,10 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - uses: actions/setup-go@v3
-        with:
-          go-version: '^1.20.0'
-          cache: true
+      # - uses: actions/setup-go@v3
+      #   with:
+      #     go-version: '^1.20.0'
+      #     cache: true
 
       - name: template melange and apko configs
         run: |

From aa8acd27c462129efcb18555e0f035d065283526 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 17:51:22 +0000
Subject: [PATCH 26/72] one more test

---
 .github/workflows/build-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 864b597831..fd5bd45d84 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -300,7 +300,7 @@ jobs:
         run: echo "cache_dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
 
       # - uses: chainguard-dev/actions/melange-build@main
-      - uses: sgalsaleh/chainguard-dev-actions/melange-build@4824ef7293e21e9d66749bd24cfd2a11f9d05dfa
+      - uses: sgalsaleh/chainguard-dev-actions/melange-build@59df1adb1d6ab2231335b89dd823fd4b3d9d8f9b
         with:
           config: deploy/melange.yaml
           archs: x86_64

From c855c454f87cb73a49643dadc9aa7d9a3b355c05 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 17:59:08 +0000
Subject: [PATCH 27/72] one more test

---
 .github/workflows/build-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index fd5bd45d84..233a5466b5 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -300,7 +300,7 @@ jobs:
         run: echo "cache_dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
 
       # - uses: chainguard-dev/actions/melange-build@main
-      - uses: sgalsaleh/chainguard-dev-actions/melange-build@59df1adb1d6ab2231335b89dd823fd4b3d9d8f9b
+      - uses: sgalsaleh/chainguard-dev-actions/melange-build@fb61b5d8f7aff4c207952cbabcd450eb9ee79380
         with:
           config: deploy/melange.yaml
           archs: x86_64

From 7e0c5d4317f6eb68fa5acdbbf2458bde965ff049 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 18:04:15 +0000
Subject: [PATCH 28/72] one more test

---
 .github/workflows/build-test.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 233a5466b5..eb8a3bf253 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -300,7 +300,7 @@ jobs:
         run: echo "cache_dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
 
       # - uses: chainguard-dev/actions/melange-build@main
-      - uses: sgalsaleh/chainguard-dev-actions/melange-build@fb61b5d8f7aff4c207952cbabcd450eb9ee79380
+      - uses: sgalsaleh/chainguard-dev-actions/melange-build@748ec4bac39eefb5d008486b348fe1f552013f57
         with:
           config: deploy/melange.yaml
           archs: x86_64

From e54b50d402111292dd0e2c5402aea83729a5ae6e Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 18:18:52 +0000
Subject: [PATCH 29/72] test again without cache

---
 .github/workflows/build-test.yaml | 14 --------------
 deploy/melange.yaml.tmpl          |  2 --
 2 files changed, 16 deletions(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index eb8a3bf253..a0406556d5 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -283,19 +283,6 @@ jobs:
           envsubst "${GIT_TAG}" < deploy/melange.yaml.tmpl > deploy/melange.yaml
           envsubst "${GIT_TAG}" < deploy/apko.yaml.tmpl > deploy/apko.yaml
 
-      # use chainguard-dev/actions/melange-build@main action below instead when caching support is added
-      # - uses: chainguard-dev/actions/setup-melange@main
-      # - uses: chainguard-dev/actions/melange-keygen@main
-      #   with:
-      #     signing-key-path: ${{ github.workspace }}/melange.rsa
-
-      # - name: build with melange
-      #   run: |
-      #     melange build deploy/melange.yaml \
-      #       --arch x86_64 \
-      #       --signing-key ${{ github.workspace }}/melange.rsa \
-      #       --cache-dir "$(go env GOMODCACHE)"
-
       - id: cache-dir
         run: echo "cache_dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
 
@@ -305,7 +292,6 @@ jobs:
           config: deploy/melange.yaml
           archs: x86_64
           sign-with-temporary-key: true
-          cache-dir: ${{ steps.cache-dir.outputs.cache_dir }}
 
       - uses: chainguard-images/actions/apko-publish@main
         with:
diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index d2a2ad7523..e72b4f3d14 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -19,8 +19,6 @@ environment:
       - go
       - nodejs
       - yarn
-  environment:
-    GOMODCACHE: '/var/cache/melange'
 
 pipeline:
   - runs: |

From 8c7346c93a021d6846ea7222d5cffdb1eb3034e2 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 21:49:54 +0000
Subject: [PATCH 30/72] use melange+apko to build kotsadm image on
 merge/release

---
 .../build-push-kotsadm-image/action.yml       | 38 +++++++++++++++++
 .github/workflows/build-test.yaml             | 30 ++------------
 .github/workflows/release.yaml                | 41 +++++++++----------
 Makefile                                      |  5 ---
 deploy/melange.yaml.tmpl                      |  2 +
 migrations/Makefile                           |  3 --
 6 files changed, 62 insertions(+), 57 deletions(-)
 create mode 100644 .github/actions/build-push-kotsadm-image/action.yml

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
new file mode 100644
index 0000000000..64bd0a2307
--- /dev/null
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -0,0 +1,38 @@
+name: 'Build and push kotsadm image'
+description: 'Composite action for building and pushing kotsadm image'
+inputs:
+  image-name:
+    description: 'Full destination kotsadm image name'
+    required: true
+
+  git-tag:
+    description: 'Git tag'
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: template melange and apko configs
+      shell: bash
+      run: |
+        export GIT_TAG=${{ inputs.git-tag }}
+        envsubst "${GIT_TAG}" < deploy/melange.yaml.tmpl > deploy/melange.yaml
+        envsubst "${GIT_TAG}" < deploy/apko.yaml.tmpl > deploy/apko.yaml
+
+    - id: cache-dir
+      shell: bash
+      run: echo "cache_dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
+
+    - uses: chainguard-dev/actions/melange-build@main
+      with:
+        config: deploy/melange.yaml
+        archs: x86_64
+        sign-with-temporary-key: true
+        cache-dir: ${{ steps.cache-dir.outputs.cache_dir }}
+
+    - uses: chainguard-images/actions/apko-publish@main
+      with:
+        config: deploy/apko.yaml
+        archs: x86_64
+        tag: ${{ inputs.image-name }}
+        vcs-url: true
diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index a0406556d5..a47bd29a40 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -271,34 +271,10 @@ jobs:
       contents: read
     steps:
       - uses: actions/checkout@v3
-
-      # - uses: actions/setup-go@v3
-      #   with:
-      #     go-version: '^1.20.0'
-      #     cache: true
-
-      - name: template melange and apko configs
-        run: |
-          export GIT_TAG=${{ needs.generate-tag.outputs.tag }}
-          envsubst "${GIT_TAG}" < deploy/melange.yaml.tmpl > deploy/melange.yaml
-          envsubst "${GIT_TAG}" < deploy/apko.yaml.tmpl > deploy/apko.yaml
-
-      - id: cache-dir
-        run: echo "cache_dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
-
-      # - uses: chainguard-dev/actions/melange-build@main
-      - uses: sgalsaleh/chainguard-dev-actions/melange-build@748ec4bac39eefb5d008486b348fe1f552013f57
-        with:
-          config: deploy/melange.yaml
-          archs: x86_64
-          sign-with-temporary-key: true
-
-      - uses: chainguard-images/actions/apko-publish@main
+      - uses: ./.github/actions/build-push-kotsadm-image
         with:
-          config: deploy/apko.yaml
-          archs: x86_64
-          tag: ttl.sh/automated-${{ github.run_id }}/kotsadm:24h
-          vcs-url: true
+          image-name: ttl.sh/automated-${{ github.run_id }}/kotsadm:24h
+          git-tag: ${{ needs.generate-tag.outputs.tag }}
 
 
   build-kots-helm:
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 0c8b480a81..a1f222d7c7 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -142,11 +142,10 @@ jobs:
     - name: Build tagged release
       working-directory: ./kurl_proxy
       run: |
-        docker build --pull -f deploy/Dockerfile -t "kotsadm/kurl-proxy:$GIT_TAG" -t kotsadm/kurl-proxy:v0.0.0-nightly .
+        docker build --pull -f deploy/Dockerfile -t "kotsadm/kurl-proxy:$GIT_TAG" .
         docker push "kotsadm/kurl-proxy:$GIT_TAG"
-        docker push kotsadm/kurl-proxy:v0.0.0-nightly
 
-  build-go-api:
+  build-kots:
     runs-on: ubuntu-20.04
     needs: [build-web, generate-tag]
     steps:
@@ -169,32 +168,27 @@ jobs:
       with:
         name: web
         path: ./web/dist
-    - name: Build Go API
+    - name: Build KOTS
       env:
         GIT_TAG: ${{ needs.generate-tag.outputs.tag }}
         SCOPE_DSN_PUBLIC: ""
-      run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make ci-test kots build
+      run: mapfile -t envs < <(grep -v '#.*' < .image.env) && export "${envs[@]}" && make ci-test kots
     - name: Upload Go API artifact
       uses: actions/upload-artifact@v3
       with:
-        name: go_api
-        path: ./bin
+        name: kots
+        path: ./bin/kots
 
-  release-go-api-tagged:
+  build-kotsadm:
     runs-on: ubuntu-20.04
-    needs: [build-go-api, generate-tag]
+    needs: [generate-tag]
     steps:
     - name: Checkout
-      uses: actions/checkout@v4
-    - name: Download go_api artifact
-      uses: actions/download-artifact@v3
+      uses: actions/checkout@v3
+    - uses: ./.github/actions/build-push-kotsadm-image
       with:
-        name: go_api
-        path: ./bin
-    - name: Add executable permissions
-      run: |
-        chmod a+x ./bin/kotsadm
-        chmod a+x ./bin/kots
+        image-name: kotsadm/kotsadm:${{ needs.generate-tag.outputs.tag }}
+        git-tag: ${{ needs.generate-tag.outputs.tag }}
     - uses: azure/docker-login@v1
       env:
         DOCKER_CONFIG: ./.docker
@@ -271,7 +265,7 @@ jobs:
 
   generate-kurl-addon:
     runs-on: ubuntu-20.04
-    needs: [ generate-tag, build-kurl-proxy, build-schema-migrations, release-go-api-tagged ]
+    needs: [ generate-tag, build-kurl-proxy, build-schema-migrations, build-kots, build-kotsadm ]
     outputs:
       addon_package_url: ${{ steps.addon-generate.outputs.addon_package_url }}
     env:
@@ -289,7 +283,7 @@ jobs:
       - name: download kots binary
         uses: actions/download-artifact@v3
         with:
-          name: go_api
+          name: kots
           path: bin/
       - name: prepare kots binary executable
         run: |
@@ -301,6 +295,7 @@ jobs:
           addon_version: ${{ steps.vars.outputs.addon_version }}
           s3_prefix: "${{ github.ref_type != 'branch' && '' || 'test/' }}"
           kotsadm_binary_override: bin/kots.tar.gz
+
   # only run validate-kurl-addon if changes to "deploy/kurl/kotsadm/template/**"
   kurl-addon-changes-filter:
     runs-on: ubuntu-20.04
@@ -315,6 +310,7 @@ jobs:
             kurl-addon:
               - 'deploy/kurl/kotsadm/template/**'
               - 'deploy/kurl/kotsadm/testgrid-os-spec.yaml'
+
   validate-kurl-addon:
     runs-on: ubuntu-20.04
     if: ${{ github.ref_type != 'branch' || needs.kurl-addon-changes-filter.outputs.ok-to-test == 'true' }}
@@ -333,6 +329,7 @@ jobs:
           addon_package_url: "${{ needs.generate-kurl-addon.outputs.addon_package_url }}"
           priority: ${{ github.ref_type != 'branch' && '1' || '0' }}
           testgrid_api_token: ${{ secrets.TESTGRID_PROD_API_TOKEN }}
+
   publish-kurl-addon:
     runs-on: ubuntu-20.04
     if: ${{ github.ref_type != 'branch' }}
@@ -377,7 +374,7 @@ jobs:
   build-airgap:
     runs-on: ubuntu-20.04
     if: github.ref_type != 'branch'
-    needs: [release-go-api-tagged, goreleaser, build-schema-migrations, generate-tag]
+    needs: [build-kotsadm, goreleaser, build-schema-migrations, generate-tag]
     steps:
     - name: Download migrations
       uses: actions/download-artifact@v3
@@ -435,7 +432,7 @@ jobs:
 
   regression-test:
     if: github.ref_type == 'branch'
-    needs: [ regression-test-setup, generate-tag, build-go-api, release-go-api-tagged, generate-kurl-addon ]
+    needs: [ regression-test-setup, generate-tag, build-kots, build-kotsadm, generate-kurl-addon ]
     uses: ./.github/workflows/regression.yaml
     with:
       version_tag_old: ${{ needs.regression-test-setup.outputs.last_release_tag }}
diff --git a/Makefile b/Makefile
index 1c0ee04e2f..45e10ed58d 100644
--- a/Makefile
+++ b/Makefile
@@ -133,14 +133,9 @@ build-alpha:
 
 .PHONY: build-release
 build-release:
-	docker build --pull -f deploy/Dockerfile --build-arg version=${GIT_TAG} -t kotsadm/kotsadm:${GIT_TAG} .
-	docker push kotsadm/kotsadm:${GIT_TAG}
 	mkdir -p bin/docker-archive/kotsadm
 	skopeo copy docker-daemon:kotsadm/kotsadm:${GIT_TAG} docker-archive:bin/docker-archive/kotsadm/${GIT_TAG}
 
-	docker tag kotsadm/kotsadm:${GIT_TAG} kotsadm/kotsadm:v0.0.0-nightly
-	docker push kotsadm/kotsadm:v0.0.0-nightly
-
 	docker build --pull -f deploy/dex.Dockerfile -t kotsadm/dex:${DEX_TAG} --build-arg TAG=${DEX_TAG} .
 	docker push kotsadm/dex:${DEX_TAG}
 	mkdir -p bin/docker-archive/dex
diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index e72b4f3d14..d2a2ad7523 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -19,6 +19,8 @@ environment:
       - go
       - nodejs
       - yarn
+  environment:
+    GOMODCACHE: '/var/cache/melange'
 
 pipeline:
   - runs: |
diff --git a/migrations/Makefile b/migrations/Makefile
index 79ffcee70d..011acb5480 100644
--- a/migrations/Makefile
+++ b/migrations/Makefile
@@ -9,10 +9,7 @@ schema-alpha: build_schema
 
 .PHONY: schema-release
 schema-release: IMAGE = kotsadm/${PROJECT_NAME}:${GIT_TAG}
-schema-release: NIGHTLY_IMAGE = kotsadm/${PROJECT_NAME}:v0.0.0-nightly
 schema-release: build_schema
-	docker tag ${IMAGE} ${NIGHTLY_IMAGE}
-	docker push ${NIGHTLY_IMAGE}
 	mkdir -p bin/docker-archive/${PROJECT_NAME}
 	skopeo copy docker-daemon:kotsadm/${PROJECT_NAME}:${GIT_TAG} docker-archive:bin/docker-archive/${PROJECT_NAME}/${GIT_TAG}
 	mkdir -p bin/docker-archive/rqlite

From c9362c408846891a25ef2ddafb43686fe3f44bb9 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 21 Nov 2023 22:06:16 +0000
Subject: [PATCH 31/72] refactor

---
 .github/workflows/build-test.yaml | 66 +++++++++++++++----------------
 1 file changed, 33 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index a47bd29a40..aced0a405a 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -264,7 +264,7 @@ jobs:
           path: ./bin/kots
 
 
-  build-push-kotsadm-image:
+  build-kotsadm:
     runs-on: ubuntu-20.04
     needs: [ can-run-ci, generate-tag ]
     permissions:
@@ -488,7 +488,7 @@ jobs:
   validate-kurl-addon:
     runs-on: ubuntu-20.04
     if: ${{ needs.kurl-addon-changes-filter.outputs.ok-to-test == 'true' }}
-    needs: [ can-run-ci, enable-tests, generate-tag, kurl-addon-changes-filter, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-dex ]
+    needs: [ can-run-ci, enable-tests, generate-tag, kurl-addon-changes-filter, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-dex ]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -552,7 +552,7 @@ jobs:
 
   validate-existing-online-install-minimal:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -587,7 +587,7 @@ jobs:
 
   validate-smoke-test:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -626,7 +626,7 @@ jobs:
 
   validate-minimal-rbac:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -738,7 +738,7 @@ jobs:
 
   validate-backup-and-restore:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -777,7 +777,7 @@ jobs:
 
   validate-no-required-config:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -817,7 +817,7 @@ jobs:
 
   validate-strict-preflight-checks:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -926,7 +926,7 @@ jobs:
 
   validate-config:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -966,7 +966,7 @@ jobs:
 
   validate-version-history-pagination:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1005,7 +1005,7 @@ jobs:
 
   validate-change-license:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1045,7 +1045,7 @@ jobs:
 
   validate-minimal-rbac-override:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1221,7 +1221,7 @@ jobs:
 
   validate-multi-namespace:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1333,7 +1333,7 @@ jobs:
 
   validate-kots-pull:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1443,7 +1443,7 @@ jobs:
 
   validate-app-version-label:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1601,7 +1601,7 @@ jobs:
 
   validate-helm-install-order:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1700,7 +1700,7 @@ jobs:
 
   validate-no-redeploy-on-restart:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1822,7 +1822,7 @@ jobs:
 
   validate-kubernetes-installer-preflight:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -1959,7 +1959,7 @@ jobs:
 
   validate-kots-push-images-anonymous:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -1980,7 +1980,7 @@ jobs:
 
   validate-kots-admin-console-generate-manifests:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -2175,7 +2175,7 @@ jobs:
 
   validate-min-kots-version:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-e2e, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
+    needs: [ enable-tests, can-run-ci, build-e2e, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
     env:
       APP_SLUG: min-kots-version
     strategy:
@@ -2245,7 +2245,7 @@ jobs:
 
   validate-target-kots-version:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-e2e, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
+    needs: [ enable-tests, can-run-ci, build-e2e, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
     strategy:
       fail-fast: false
       matrix:
@@ -2312,7 +2312,7 @@ jobs:
 
   validate-range-kots-version:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-e2e, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
+    needs: [ enable-tests, can-run-ci, build-e2e, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite, generate-tag ]
     strategy:
       fail-fast: false
       matrix:
@@ -2379,7 +2379,7 @@ jobs:
 
   validate-kots-upgrade:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, cmx-versions, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -2518,7 +2518,7 @@ jobs:
 
   validate-kots-helm-release-secret-migration:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -2705,7 +2705,7 @@ jobs:
 
   validate-multi-app-backup-and-restore:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -2745,7 +2745,7 @@ jobs:
 
   validate-multi-app-install:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -2785,7 +2785,7 @@ jobs:
 
   validate-airgap-smoke-test:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-e2e, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -2826,7 +2826,7 @@ jobs:
 
   validate-remove-app:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -3005,7 +3005,7 @@ jobs:
 
   validate-registry-check:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -3100,7 +3100,7 @@ jobs:
 
   validate-native-helm-v2:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -3326,7 +3326,7 @@ jobs:
 
   validate-deployment-orchestration:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-kots, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kots, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:
@@ -3494,7 +3494,7 @@ jobs:
 
   validate-replicated-sdk:
     runs-on: ubuntu-20.04
-    needs: [ enable-tests, can-run-ci, build-push-kotsadm-image, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
+    needs: [ enable-tests, can-run-ci, build-kotsadm, build-kurl-proxy, build-migrations, push-minio, push-rqlite ]
     strategy:
       fail-fast: false
       matrix:

From 657a4189e4a4b48a2ab608abe6c7fb2a4b964a2d Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 22 Nov 2023 14:24:10 +0000
Subject: [PATCH 32/72] add kubectl-1.28

---
 deploy/apko.yaml.tmpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl
index 241a90075d..af1f846481 100644
--- a/deploy/apko.yaml.tmpl
+++ b/deploy/apko.yaml.tmpl
@@ -10,10 +10,10 @@ contents:
 
     # All currently supported kubectl versions.
     # TODO: this requires manual intervention whenever there are new kubectl releases.
-    - kubectl-1.24
     - kubectl-1.25
     - kubectl-1.26
     - kubectl-1.27
+    - kubectl-1.28
 
     - bash
     - busybox

From d0b37f36ffd57c338f6086d57685d096f1a55f42 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 22 Nov 2023 15:21:50 +0000
Subject: [PATCH 33/72] fix symlinks

---
 deploy/melange.yaml.tmpl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index d2a2ad7523..cffc98ca0a 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -74,8 +74,8 @@ pipeline:
       ln -s /usr/bin/kustomize ${DESTDIR}/usr/local/bin/kustomize5
 
       # TODO: this requires manual intervention whenever kubectl bumps its major version
-      ln -s /usr/bin/kubectl-1.24 ${DESTDIR}/usr/local/bin/kubectl-v1.24
       ln -s /usr/bin/kubectl-1.25 ${DESTDIR}/usr/local/bin/kubectl-v1.25
       ln -s /usr/bin/kubectl-1.26 ${DESTDIR}/usr/local/bin/kubectl-v1.26
       ln -s /usr/bin/kubectl-1.27 ${DESTDIR}/usr/local/bin/kubectl-v1.27
-      ln -s /usr/bin/kubectl-1.27 ${DESTDIR}/usr/local/bin/kubectl
+      ln -s /usr/bin/kubectl-1.28 ${DESTDIR}/usr/local/bin/kubectl-v1.28
+      ln -s /usr/bin/kubectl-1.28 ${DESTDIR}/usr/local/bin/kubectl

From 1a9186a29d294ad87e720951e75df6867a622d2a Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Sat, 25 Nov 2023 15:16:54 +0000
Subject: [PATCH 34/72] fix envsubset

---
 .github/actions/build-push-kotsadm-image/action.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 64bd0a2307..a71419fe21 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -16,8 +16,8 @@ runs:
       shell: bash
       run: |
         export GIT_TAG=${{ inputs.git-tag }}
-        envsubst "${GIT_TAG}" < deploy/melange.yaml.tmpl > deploy/melange.yaml
-        envsubst "${GIT_TAG}" < deploy/apko.yaml.tmpl > deploy/apko.yaml
+        envsubst '${GIT_TAG}' < deploy/melange.yaml.tmpl > deploy/melange.yaml
+        envsubst '${GIT_TAG}' < deploy/apko.yaml.tmpl > deploy/apko.yaml
 
     - id: cache-dir
       shell: bash

From b24a7f0a06ff9b81d9f5ab53452d8f77bb1cd55b Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 27 Nov 2023 14:50:12 +0000
Subject: [PATCH 35/72] test regression tests

---
 .github/workflows/release.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index a1f222d7c7..0ec2ecb0e0 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -6,6 +6,7 @@ on:
       - "v*.*.*"
     branches:
       - main
+      - integration-chainguard
 
 jobs:
   generate-tag:

From 222dc63433c74969ee4e00464a980f3b867c88d8 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 27 Nov 2023 15:04:17 +0000
Subject: [PATCH 36/72] docker login before

---
 .github/actions/build-push-kotsadm-image/action.yml | 12 ++++++++++++
 .github/workflows/release.yaml                      | 10 ++++++----
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index a71419fe21..df0696675a 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -9,6 +9,16 @@ inputs:
     description: 'Git tag'
     required: true
 
+  registry-username:
+    description: 'Username to login to registry'
+    default: ''
+    required: false
+
+  registry-password:
+    description: 'Password to login to registry'
+    default: ''
+    required: false
+
 runs:
   using: "composite"
   steps:
@@ -36,3 +46,5 @@ runs:
         archs: x86_64
         tag: ${{ inputs.image-name }}
         vcs-url: true
+        generic-user: ${{ inputs.registry-username }}
+        generic-pass: ${{ inputs.registry-password }}
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 0ec2ecb0e0..c07ab8d43b 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -186,16 +186,18 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v3
-    - uses: ./.github/actions/build-push-kotsadm-image
-      with:
-        image-name: kotsadm/kotsadm:${{ needs.generate-tag.outputs.tag }}
-        git-tag: ${{ needs.generate-tag.outputs.tag }}
     - uses: azure/docker-login@v1
       env:
         DOCKER_CONFIG: ./.docker
       with:
         username: ${{ secrets.DOCKERHUB_USER }}
         password: ${{ secrets.DOCKERHUB_PASSWORD }}
+    - uses: ./.github/actions/build-push-kotsadm-image
+      with:
+        image-name: kotsadm/kotsadm:${{ needs.generate-tag.outputs.tag }}
+        git-tag: ${{ needs.generate-tag.outputs.tag }}
+        # registry-username: ${{ secrets.DOCKERHUB_USER }}
+        # registry-password: ${{ secrets.DOCKERHUB_PASSWORD }}
     - name: Build tagged release
       env:
         GIT_TAG: ${{ needs.generate-tag.outputs.tag }}

From 124a5fd95fbe544d9262f2adb92f21f1221ccb8c Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 27 Nov 2023 15:15:05 +0000
Subject: [PATCH 37/72] username and password for apko

---
 .github/workflows/release.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index c07ab8d43b..2c6458464c 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -186,18 +186,18 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v3
+    - uses: ./.github/actions/build-push-kotsadm-image
+      with:
+        image-name: kotsadm/kotsadm:${{ needs.generate-tag.outputs.tag }}
+        git-tag: ${{ needs.generate-tag.outputs.tag }}
+        registry-username: ${{ secrets.DOCKERHUB_USER }}
+        registry-password: ${{ secrets.DOCKERHUB_PASSWORD }}
     - uses: azure/docker-login@v1
       env:
         DOCKER_CONFIG: ./.docker
       with:
         username: ${{ secrets.DOCKERHUB_USER }}
         password: ${{ secrets.DOCKERHUB_PASSWORD }}
-    - uses: ./.github/actions/build-push-kotsadm-image
-      with:
-        image-name: kotsadm/kotsadm:${{ needs.generate-tag.outputs.tag }}
-        git-tag: ${{ needs.generate-tag.outputs.tag }}
-        # registry-username: ${{ secrets.DOCKERHUB_USER }}
-        # registry-password: ${{ secrets.DOCKERHUB_PASSWORD }}
     - name: Build tagged release
       env:
         GIT_TAG: ${{ needs.generate-tag.outputs.tag }}

From ec504dbf5b8a53209565ec237f04a897130e982e Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 27 Nov 2023 15:48:49 +0000
Subject: [PATCH 38/72] hack to build kotsadm image and trigger tests

---
 .github/workflows/release.yaml | 6 +++---
 Makefile                       | 3 +++
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 2c6458464c..80c2d0885d 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -188,10 +188,10 @@ jobs:
       uses: actions/checkout@v3
     - uses: ./.github/actions/build-push-kotsadm-image
       with:
-        image-name: kotsadm/kotsadm:${{ needs.generate-tag.outputs.tag }}
+        image-name: ttl.sh/salah/kotsadm:${{ needs.generate-tag.outputs.tag }}
         git-tag: ${{ needs.generate-tag.outputs.tag }}
-        registry-username: ${{ secrets.DOCKERHUB_USER }}
-        registry-password: ${{ secrets.DOCKERHUB_PASSWORD }}
+        # registry-username: ${{ secrets.DOCKERHUB_USER }}
+        # registry-password: ${{ secrets.DOCKERHUB_PASSWORD }}
     - uses: azure/docker-login@v1
       env:
         DOCKER_CONFIG: ./.docker
diff --git a/Makefile b/Makefile
index 45e10ed58d..b218373216 100644
--- a/Makefile
+++ b/Makefile
@@ -133,6 +133,9 @@ build-alpha:
 
 .PHONY: build-release
 build-release:
+	docker tag ttl.sh/salah/kotsadm:${GIT_TAG} kotsadm/kotsadm:${GIT_TAG}
+	docker push kotsadm/kotsadm:${GIT_TAG}
+
 	mkdir -p bin/docker-archive/kotsadm
 	skopeo copy docker-daemon:kotsadm/kotsadm:${GIT_TAG} docker-archive:bin/docker-archive/kotsadm/${GIT_TAG}
 

From 7f61576403c9aac3d830df7e45e7a50023c18b48 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 27 Nov 2023 16:01:18 +0000
Subject: [PATCH 39/72] one more try

---
 Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile b/Makefile
index b218373216..a88aeb711d 100644
--- a/Makefile
+++ b/Makefile
@@ -133,6 +133,7 @@ build-alpha:
 
 .PHONY: build-release
 build-release:
+	docker pull ttl.sh/salah/kotsadm:${GIT_TAG}
 	docker tag ttl.sh/salah/kotsadm:${GIT_TAG} kotsadm/kotsadm:${GIT_TAG}
 	docker push kotsadm/kotsadm:${GIT_TAG}
 

From d005ce7c4ee05714099c9e7ae50bbe4b31b91e69 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 27 Nov 2023 16:19:55 +0000
Subject: [PATCH 40/72] one more try

---
 .github/workflows/regression.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/regression.yaml b/.github/workflows/regression.yaml
index 0373bc1ccf..8f651aa58b 100644
--- a/.github/workflows/regression.yaml
+++ b/.github/workflows/regression.yaml
@@ -70,10 +70,10 @@ jobs:
           token: ${{ secrets.E2E_GH_PAT }}
           path: automation
           ref: main
-      - name: Download go_api artifact
+      - name: Download kots binary
         uses: actions/download-artifact@v3
         with:
-          name: go_api
+          name: kots
           path: ./automation/jumpbox/bin
       - name: Download kotsadm image
         uses: actions/download-artifact@v3

From 8c3a76b60e17f7c4311ee14ae738331388675627 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 27 Nov 2023 19:40:40 +0000
Subject: [PATCH 41/72] use creds to push to dockerhub

---
 .github/workflows/release.yaml | 6 +++---
 Makefile                       | 4 ----
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 80c2d0885d..5f843aea54 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -188,10 +188,10 @@ jobs:
       uses: actions/checkout@v3
     - uses: ./.github/actions/build-push-kotsadm-image
       with:
-        image-name: ttl.sh/salah/kotsadm:${{ needs.generate-tag.outputs.tag }}
+        image-name: index.docker.io/kotsadm/kotsadm:${{ needs.generate-tag.outputs.tag }}
         git-tag: ${{ needs.generate-tag.outputs.tag }}
-        # registry-username: ${{ secrets.DOCKERHUB_USER }}
-        # registry-password: ${{ secrets.DOCKERHUB_PASSWORD }}
+        registry-username: ${{ secrets.DOCKERHUB_USER }}
+        registry-password: ${{ secrets.DOCKERHUB_PASSWORD }}
     - uses: azure/docker-login@v1
       env:
         DOCKER_CONFIG: ./.docker
diff --git a/Makefile b/Makefile
index a88aeb711d..45e10ed58d 100644
--- a/Makefile
+++ b/Makefile
@@ -133,10 +133,6 @@ build-alpha:
 
 .PHONY: build-release
 build-release:
-	docker pull ttl.sh/salah/kotsadm:${GIT_TAG}
-	docker tag ttl.sh/salah/kotsadm:${GIT_TAG} kotsadm/kotsadm:${GIT_TAG}
-	docker push kotsadm/kotsadm:${GIT_TAG}
-
 	mkdir -p bin/docker-archive/kotsadm
 	skopeo copy docker-daemon:kotsadm/kotsadm:${GIT_TAG} docker-archive:bin/docker-archive/kotsadm/${GIT_TAG}
 

From 17c2d820c9b842079829bdf21949ae61eae46627 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Mon, 27 Nov 2023 21:19:37 +0000
Subject: [PATCH 42/72] copy from docker

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 45e10ed58d..b2c6216431 100644
--- a/Makefile
+++ b/Makefile
@@ -134,7 +134,7 @@ build-alpha:
 .PHONY: build-release
 build-release:
 	mkdir -p bin/docker-archive/kotsadm
-	skopeo copy docker-daemon:kotsadm/kotsadm:${GIT_TAG} docker-archive:bin/docker-archive/kotsadm/${GIT_TAG}
+	skopeo copy docker://kotsadm/kotsadm:${GIT_TAG} docker-archive:bin/docker-archive/kotsadm/${GIT_TAG}
 
 	docker build --pull -f deploy/dex.Dockerfile -t kotsadm/dex:${DEX_TAG} --build-arg TAG=${DEX_TAG} .
 	docker push kotsadm/dex:${DEX_TAG}

From 134dcc07e88855c698b1ee61a61730827c7bd068 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 28 Nov 2023 18:07:23 +0000
Subject: [PATCH 43/72] include older kubectl versions

---
 .../build-push-kotsadm-image/action.yml       | 27 +++++++++++++++++++
 .github/workflows/build-test.yaml             |  2 +-
 .github/workflows/release.yaml                |  2 ++
 deploy/apko.yaml.tmpl                         |  6 +++++
 deploy/melange.yaml.tmpl                      |  6 +++++
 5 files changed, 42 insertions(+), 1 deletion(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index df0696675a..6cda8ec603 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -22,6 +22,33 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
+      with:
+        workload_identity_provider: ${{secrets.CHAINGUARD_GCP_WIF_POOL}}
+        service_account: ${{secrets.CHAINGUARD_GCP_SA}}
+
+    - uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1
+      with:
+        project_id: ${{secrets.CHAINGUARD_GCP_PROJECT_ID}}
+
+    - name: setup packages gcsfuse
+      env:
+        BUCKET: replicated-apk-registry
+      run: |
+        # Set up a gcsfuse RO mount to the bucket containing private packages. This is a cheap and
+        # cheerful way to get access to objects we need, without having to fetch all of them.
+        mkdir -p /gcsfuse/apk-repo
+        gcsfuse -o ro --implicit-dirs --only-dir os ${BUCKET} /gcsfuse/apk-repo
+
+        # Symlink the gcsfuse mount to ./packages/$arch/*.apk
+        mkdir -p ./packages/x86_64
+        ln -s /gcsfuse/apk-repo/x86_64/*.apk ./packages/x86_64/
+
+        # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
+        cp /gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/
+
+        ls -lR ./packages/
+
     - name: template melange and apko configs
       shell: bash
       run: |
diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index aced0a405a..7336836a2a 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -268,7 +268,7 @@ jobs:
     runs-on: ubuntu-20.04
     needs: [ can-run-ci, generate-tag ]
     permissions:
-      contents: read
+      id-token: write # required to be able to assume the GCP SA identity to pull Chainguard packages.
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/build-push-kotsadm-image
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 5f843aea54..356354ee9c 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -183,6 +183,8 @@ jobs:
   build-kotsadm:
     runs-on: ubuntu-20.04
     needs: [generate-tag]
+    permissions:
+      id-token: write # required to be able to assume the GCP SA identity to pull Chainguard packages.
     steps:
     - name: Checkout
       uses: actions/checkout@v3
diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl
index af1f846481..276ab70a76 100644
--- a/deploy/apko.yaml.tmpl
+++ b/deploy/apko.yaml.tmpl
@@ -10,6 +10,12 @@ contents:
 
     # All currently supported kubectl versions.
     # TODO: this requires manual intervention whenever there are new kubectl releases.
+    - kubectl-1.19
+    - kubectl-1.20
+    - kubectl-1.21
+    - kubectl-1.22
+    - kubectl-1.23
+    - kubectl-1.24
     - kubectl-1.25
     - kubectl-1.26
     - kubectl-1.27
diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index cffc98ca0a..f86388720f 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -74,6 +74,12 @@ pipeline:
       ln -s /usr/bin/kustomize ${DESTDIR}/usr/local/bin/kustomize5
 
       # TODO: this requires manual intervention whenever kubectl bumps its major version
+      ln -s /usr/bin/kubectl-1.19 ${DESTDIR}/usr/local/bin/kubectl-v1.19
+      ln -s /usr/bin/kubectl-1.20 ${DESTDIR}/usr/local/bin/kubectl-v1.20
+      ln -s /usr/bin/kubectl-1.21 ${DESTDIR}/usr/local/bin/kubectl-v1.21
+      ln -s /usr/bin/kubectl-1.22 ${DESTDIR}/usr/local/bin/kubectl-v1.22
+      ln -s /usr/bin/kubectl-1.23 ${DESTDIR}/usr/local/bin/kubectl-v1.23
+      ln -s /usr/bin/kubectl-1.24 ${DESTDIR}/usr/local/bin/kubectl-v1.24
       ln -s /usr/bin/kubectl-1.25 ${DESTDIR}/usr/local/bin/kubectl-v1.25
       ln -s /usr/bin/kubectl-1.26 ${DESTDIR}/usr/local/bin/kubectl-v1.26
       ln -s /usr/bin/kubectl-1.27 ${DESTDIR}/usr/local/bin/kubectl-v1.27

From ef344c44a3afda1462f68b3b7a822febe45b0ae1 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 28 Nov 2023 20:02:59 +0000
Subject: [PATCH 44/72] updates

---
 .github/actions/build-push-kotsadm-image/action.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 6cda8ec603..72a4bfe89f 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -24,12 +24,12 @@ runs:
   steps:
     - uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
       with:
-        workload_identity_provider: ${{secrets.CHAINGUARD_GCP_WIF_POOL}}
-        service_account: ${{secrets.CHAINGUARD_GCP_SA}}
+        workload_identity_provider: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }}
+        service_account: ${{ secrets.CHAINGUARD_GCP_SA }}
 
     - uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1
       with:
-        project_id: ${{secrets.CHAINGUARD_GCP_PROJECT_ID}}
+        project_id: ${{ secrets.CHAINGUARD_GCP_PROJECT_ID }}
 
     - name: setup packages gcsfuse
       env:

From bf55a46b92bc7725c007c08de019cd2abb9d3624 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 28 Nov 2023 20:11:14 +0000
Subject: [PATCH 45/72] pass secrets as inputs

---
 .../build-push-kotsadm-image/action.yml        | 18 +++++++++++++++---
 .github/workflows/build-test.yaml              |  3 +++
 .github/workflows/release.yaml                 |  3 +++
 3 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 72a4bfe89f..aecb4bcbe1 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -1,6 +1,18 @@
 name: 'Build and push kotsadm image'
 description: 'Composite action for building and pushing kotsadm image'
 inputs:
+  chainguard-gcp-wif-pool:
+    description: 'Chainguard's GCP workload identity pool'
+    required: true
+
+  chainguard-gcp-sa:
+    description: 'Chainguard's GCP service account'
+    required: true
+
+  chainguard-gcp-project-id:
+    description: 'Chainguard's GCP project ID'
+    required: true
+
   image-name:
     description: 'Full destination kotsadm image name'
     required: true
@@ -24,12 +36,12 @@ runs:
   steps:
     - uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
       with:
-        workload_identity_provider: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }}
-        service_account: ${{ secrets.CHAINGUARD_GCP_SA }}
+        workload_identity_provider: ${{ inputs.chainguard-gcp-wif-pool }}
+        service_account: ${{ inputs.chainguard-gcp-sa }}
 
     - uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1
       with:
-        project_id: ${{ secrets.CHAINGUARD_GCP_PROJECT_ID }}
+        project_id: ${{ inputs.chainguard-gcp-project-id }}
 
     - name: setup packages gcsfuse
       env:
diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
index 7336836a2a..42acbfac93 100644
--- a/.github/workflows/build-test.yaml
+++ b/.github/workflows/build-test.yaml
@@ -273,6 +273,9 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/build-push-kotsadm-image
         with:
+          chainguard-gcp-wif-pool: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }}
+          chainguard-gcp-sa: ${{ secrets.CHAINGUARD_GCP_SA }}
+          chainguard-gcp-project-id: ${{ secrets.CHAINGUARD_GCP_PROJECT_ID }}
           image-name: ttl.sh/automated-${{ github.run_id }}/kotsadm:24h
           git-tag: ${{ needs.generate-tag.outputs.tag }}
 
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 356354ee9c..6507b453ea 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -190,6 +190,9 @@ jobs:
       uses: actions/checkout@v3
     - uses: ./.github/actions/build-push-kotsadm-image
       with:
+        chainguard-gcp-wif-pool: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }}
+        chainguard-gcp-sa: ${{ secrets.CHAINGUARD_GCP_SA }}
+        chainguard-gcp-project-id: ${{ secrets.CHAINGUARD_GCP_PROJECT_ID }}
         image-name: index.docker.io/kotsadm/kotsadm:${{ needs.generate-tag.outputs.tag }}
         git-tag: ${{ needs.generate-tag.outputs.tag }}
         registry-username: ${{ secrets.DOCKERHUB_USER }}

From 57d6fb7777e65871e1e3e93ff49d294fb932ef61 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 28 Nov 2023 20:16:21 +0000
Subject: [PATCH 46/72] fix syntax issue

---
 .github/actions/build-push-kotsadm-image/action.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index aecb4bcbe1..8f1e3ffe5a 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -2,15 +2,15 @@ name: 'Build and push kotsadm image'
 description: 'Composite action for building and pushing kotsadm image'
 inputs:
   chainguard-gcp-wif-pool:
-    description: 'Chainguard's GCP workload identity pool'
+    description: 'GCP workload identity pool for Chainguard'
     required: true
 
   chainguard-gcp-sa:
-    description: 'Chainguard's GCP service account'
+    description: 'GCP service account for Chainguard'
     required: true
 
   chainguard-gcp-project-id:
-    description: 'Chainguard's GCP project ID'
+    description: 'GCP project ID for Chainguard'
     required: true
 
   image-name:

From f698ef1959b2f7088875eb39e662b203ca8bb5ac Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Tue, 28 Nov 2023 20:30:30 +0000
Subject: [PATCH 47/72] shell: bash

---
 .github/actions/build-push-kotsadm-image/action.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 8f1e3ffe5a..88942908fa 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -46,6 +46,7 @@ runs:
     - name: setup packages gcsfuse
       env:
         BUCKET: replicated-apk-registry
+      shell: bash
       run: |
         # Set up a gcsfuse RO mount to the bucket containing private packages. This is a cheap and
         # cheerful way to get access to objects we need, without having to fetch all of them.

From 46014df7d815839a3d935a1e25a4152b7c34d071 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 14:48:00 +0000
Subject: [PATCH 48/72] gcsfuse under /tmp

---
 .github/actions/build-push-kotsadm-image/action.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 88942908fa..9e38a6baf9 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -50,15 +50,15 @@ runs:
       run: |
         # Set up a gcsfuse RO mount to the bucket containing private packages. This is a cheap and
         # cheerful way to get access to objects we need, without having to fetch all of them.
-        mkdir -p /gcsfuse/apk-repo
-        gcsfuse -o ro --implicit-dirs --only-dir os ${BUCKET} /gcsfuse/apk-repo
+        mkdir -p /tmp/gcsfuse/apk-repo
+        gcsfuse -o ro --implicit-dirs --only-dir os ${BUCKET} /tmp/gcsfuse/apk-repo
 
         # Symlink the gcsfuse mount to ./packages/$arch/*.apk
         mkdir -p ./packages/x86_64
-        ln -s /gcsfuse/apk-repo/x86_64/*.apk ./packages/x86_64/
+        ln -s /tmp/gcsfuse/apk-repo/x86_64/*.apk ./packages/x86_64/
 
         # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
-        cp /gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/
+        cp /tmp/gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/
 
         ls -lR ./packages/
 

From 49753c4338be255e05d4bd1e197ec21066f51d7f Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 14:55:08 +0000
Subject: [PATCH 49/72] install gcsfuse

---
 .github/actions/build-push-kotsadm-image/action.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 9e38a6baf9..8643cedc91 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -48,6 +48,14 @@ runs:
         BUCKET: replicated-apk-registry
       shell: bash
       run: |
+        # Install gcsfuse
+        export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
+        echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
+        curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+
+        sudo apt-get update -y
+        sudo apt-get install gcsfuse -y
+
         # Set up a gcsfuse RO mount to the bucket containing private packages. This is a cheap and
         # cheerful way to get access to objects we need, without having to fetch all of them.
         mkdir -p /tmp/gcsfuse/apk-repo

From 88797ec1e81af89c8c09e7b343e8062180806304 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 14:58:25 +0000
Subject: [PATCH 50/72] one more try

---
 .github/actions/build-push-kotsadm-image/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 8643cedc91..4b03975ec9 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -50,8 +50,8 @@ runs:
       run: |
         # Install gcsfuse
         export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
-        echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
         curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+        echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
 
         sudo apt-get update -y
         sudo apt-get install gcsfuse -y

From 09965ee35333f8c262e1e782e37631bb5bad241d Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 15:07:19 +0000
Subject: [PATCH 51/72] fix getting publish key

---
 .github/actions/build-push-kotsadm-image/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 4b03975ec9..2543a875d0 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -50,8 +50,8 @@ runs:
       run: |
         # Install gcsfuse
         export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
-        curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
         echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
+        curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
 
         sudo apt-get update -y
         sudo apt-get install gcsfuse -y

From bfb865ed04624ba05648a5b07e68f0c5128dd36e Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 15:09:00 +0000
Subject: [PATCH 52/72] one more try

---
 .github/actions/build-push-kotsadm-image/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 2543a875d0..67d4ccdd12 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -51,7 +51,7 @@ runs:
         # Install gcsfuse
         export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
         echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
-        curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg
+        curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo tee /usr/share/keyrings/cloud.google.asc
 
         sudo apt-get update -y
         sudo apt-get install gcsfuse -y

From 087de23897bbc69bdb4e909d20fe2bc77cce1696 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 15:17:19 +0000
Subject: [PATCH 53/72] test without gcsfuse

---
 .../build-push-kotsadm-image/action.yml       | 69 +++++++++----------
 1 file changed, 34 insertions(+), 35 deletions(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 67d4ccdd12..51c4f1c169 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -34,41 +34,40 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
-      with:
-        workload_identity_provider: ${{ inputs.chainguard-gcp-wif-pool }}
-        service_account: ${{ inputs.chainguard-gcp-sa }}
-
-    - uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1
-      with:
-        project_id: ${{ inputs.chainguard-gcp-project-id }}
-
-    - name: setup packages gcsfuse
-      env:
-        BUCKET: replicated-apk-registry
-      shell: bash
-      run: |
-        # Install gcsfuse
-        export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
-        echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
-        curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo tee /usr/share/keyrings/cloud.google.asc
-
-        sudo apt-get update -y
-        sudo apt-get install gcsfuse -y
-
-        # Set up a gcsfuse RO mount to the bucket containing private packages. This is a cheap and
-        # cheerful way to get access to objects we need, without having to fetch all of them.
-        mkdir -p /tmp/gcsfuse/apk-repo
-        gcsfuse -o ro --implicit-dirs --only-dir os ${BUCKET} /tmp/gcsfuse/apk-repo
-
-        # Symlink the gcsfuse mount to ./packages/$arch/*.apk
-        mkdir -p ./packages/x86_64
-        ln -s /tmp/gcsfuse/apk-repo/x86_64/*.apk ./packages/x86_64/
-
-        # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
-        cp /tmp/gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/
-
-        ls -lR ./packages/
+    # - uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
+    #   with:
+    #     workload_identity_provider: ${{ inputs.chainguard-gcp-wif-pool }}
+    #     service_account: ${{ inputs.chainguard-gcp-sa }}
+
+    # - uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1
+    #   with:
+    #     project_id: ${{ inputs.chainguard-gcp-project-id }}
+
+    # - name: setup packages gcsfuse
+    #   env:
+    #     BUCKET: replicated-apk-registry
+    #   shell: bash
+    #   run: |
+    #     # Install gcsfuse
+    #     export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
+    #     echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
+    #     curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo tee /usr/share/keyrings/cloud.google.asc
+    #     sudo apt-get update -y
+    #     sudo apt-get install gcsfuse -y
+
+    #     # Set up a gcsfuse RO mount to the bucket containing private packages. This is a cheap and
+    #     # cheerful way to get access to objects we need, without having to fetch all of them.
+    #     mkdir -p /tmp/gcsfuse/apk-repo
+    #     gcsfuse -o ro --implicit-dirs --only-dir os ${BUCKET} /tmp/gcsfuse/apk-repo
+
+    #     # Symlink the gcsfuse mount to ./packages/$arch/*.apk
+    #     mkdir -p ./packages/x86_64
+    #     ln -s /tmp/gcsfuse/apk-repo/x86_64/*.apk ./packages/x86_64/
+
+    #     # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
+    #     cp /tmp/gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/
+
+    #     ls -lR ./packages/
 
     - name: template melange and apko configs
       shell: bash

From e3b90a441ece3b345c3e6f935b88992190c50557 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 15:30:34 +0000
Subject: [PATCH 54/72] add enterprise keyring

---
 .../build-push-kotsadm-image/action.yml       | 68 +++++++++----------
 deploy/apko.yaml.tmpl                         |  1 +
 2 files changed, 35 insertions(+), 34 deletions(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 51c4f1c169..4914193b87 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -34,40 +34,40 @@ inputs:
 runs:
   using: "composite"
   steps:
-    # - uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
-    #   with:
-    #     workload_identity_provider: ${{ inputs.chainguard-gcp-wif-pool }}
-    #     service_account: ${{ inputs.chainguard-gcp-sa }}
-
-    # - uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1
-    #   with:
-    #     project_id: ${{ inputs.chainguard-gcp-project-id }}
-
-    # - name: setup packages gcsfuse
-    #   env:
-    #     BUCKET: replicated-apk-registry
-    #   shell: bash
-    #   run: |
-    #     # Install gcsfuse
-    #     export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
-    #     echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
-    #     curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo tee /usr/share/keyrings/cloud.google.asc
-    #     sudo apt-get update -y
-    #     sudo apt-get install gcsfuse -y
-
-    #     # Set up a gcsfuse RO mount to the bucket containing private packages. This is a cheap and
-    #     # cheerful way to get access to objects we need, without having to fetch all of them.
-    #     mkdir -p /tmp/gcsfuse/apk-repo
-    #     gcsfuse -o ro --implicit-dirs --only-dir os ${BUCKET} /tmp/gcsfuse/apk-repo
-
-    #     # Symlink the gcsfuse mount to ./packages/$arch/*.apk
-    #     mkdir -p ./packages/x86_64
-    #     ln -s /tmp/gcsfuse/apk-repo/x86_64/*.apk ./packages/x86_64/
-
-    #     # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
-    #     cp /tmp/gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/
-
-    #     ls -lR ./packages/
+    - uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
+      with:
+        workload_identity_provider: ${{ inputs.chainguard-gcp-wif-pool }}
+        service_account: ${{ inputs.chainguard-gcp-sa }}
+
+    - uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1
+      with:
+        project_id: ${{ inputs.chainguard-gcp-project-id }}
+
+    - name: setup packages gcsfuse
+      env:
+        BUCKET: replicated-apk-registry
+      shell: bash
+      run: |
+        # Install gcsfuse
+        export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
+        echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
+        curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo tee /usr/share/keyrings/cloud.google.asc
+        sudo apt-get update -y
+        sudo apt-get install gcsfuse -y
+
+        # Set up a gcsfuse RO mount to the bucket containing private packages. This is a cheap and
+        # cheerful way to get access to objects we need, without having to fetch all of them.
+        mkdir -p /tmp/gcsfuse/apk-repo
+        gcsfuse -o ro --implicit-dirs --only-dir os ${BUCKET} /tmp/gcsfuse/apk-repo
+
+        # Symlink the gcsfuse mount to ./packages/$arch/*.apk
+        mkdir -p ./packages/x86_64
+        ln -s /tmp/gcsfuse/apk-repo/x86_64/*.apk ./packages/x86_64/
+
+        # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
+        cp /tmp/gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/
+
+        ls -lR ./packages/
 
     - name: template melange and apko configs
       shell: bash
diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl
index 276ab70a76..1b624bd1ac 100644
--- a/deploy/apko.yaml.tmpl
+++ b/deploy/apko.yaml.tmpl
@@ -4,6 +4,7 @@ contents:
     - ./packages/
   keyring:
     - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+    - ./packages/chainguard-enterprise.rsa.pub
     - ./melange.rsa.pub
   packages:
     - kotsadm-head  # This is expected to be built locally by `melange`.

From e480d4715b8f8a4cfa1292e503ada2ee187696e1 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 15:37:51 +0000
Subject: [PATCH 55/72] add key to melange not apko

---
 deploy/apko.yaml.tmpl    | 1 -
 deploy/melange.yaml.tmpl | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl
index 1b624bd1ac..276ab70a76 100644
--- a/deploy/apko.yaml.tmpl
+++ b/deploy/apko.yaml.tmpl
@@ -4,7 +4,6 @@ contents:
     - ./packages/
   keyring:
     - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
-    - ./packages/chainguard-enterprise.rsa.pub
     - ./melange.rsa.pub
   packages:
     - kotsadm-head  # This is expected to be built locally by `melange`.
diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index f86388720f..8f38f6553a 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -12,6 +12,7 @@ environment:
       - https://packages.wolfi.dev/os
     keyring:
       - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+      - ./packages/chainguard-enterprise.rsa.pub
     packages:
       - ca-certificates-bundle
       - busybox

From 78ff36a46151c305aae5bee261d28a1986994b69 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 15:57:47 +0000
Subject: [PATCH 56/72] symlink key

---
 .github/actions/build-push-kotsadm-image/action.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 4914193b87..62b3f62fb0 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -63,6 +63,7 @@ runs:
         # Symlink the gcsfuse mount to ./packages/$arch/*.apk
         mkdir -p ./packages/x86_64
         ln -s /tmp/gcsfuse/apk-repo/x86_64/*.apk ./packages/x86_64/
+        ln -s /tmp/gcsfuse/apk-repo/chainguard-enterprise.rsa.pub ./packages/
 
         # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
         cp /tmp/gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/

From b228bfa1274a848c72e73429ab88930f2610ae82 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 16:54:13 +0000
Subject: [PATCH 57/72] one more test

---
 .github/actions/build-push-kotsadm-image/action.yml | 1 +
 deploy/melange.yaml.tmpl                            | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 62b3f62fb0..11c27adb00 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -87,6 +87,7 @@ runs:
         archs: x86_64
         sign-with-temporary-key: true
         cache-dir: ${{ steps.cache-dir.outputs.cache_dir }}
+        keyring-append: ./packages/chainguard-enterprise.rsa.pub
 
     - uses: chainguard-images/actions/apko-publish@main
       with:
diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index 8f38f6553a..f86388720f 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -12,7 +12,6 @@ environment:
       - https://packages.wolfi.dev/os
     keyring:
       - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
-      - ./packages/chainguard-enterprise.rsa.pub
     packages:
       - ca-certificates-bundle
       - busybox

From ded527687787c158e99c0f318312f4be59ff2bb6 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 17:03:05 +0000
Subject: [PATCH 58/72] one more try

---
 .github/actions/build-push-kotsadm-image/action.yml | 1 -
 deploy/apko.yaml.tmpl                               | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 11c27adb00..62b3f62fb0 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -87,7 +87,6 @@ runs:
         archs: x86_64
         sign-with-temporary-key: true
         cache-dir: ${{ steps.cache-dir.outputs.cache_dir }}
-        keyring-append: ./packages/chainguard-enterprise.rsa.pub
 
     - uses: chainguard-images/actions/apko-publish@main
       with:
diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl
index 276ab70a76..1b624bd1ac 100644
--- a/deploy/apko.yaml.tmpl
+++ b/deploy/apko.yaml.tmpl
@@ -4,6 +4,7 @@ contents:
     - ./packages/
   keyring:
     - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+    - ./packages/chainguard-enterprise.rsa.pub
     - ./melange.rsa.pub
   packages:
     - kotsadm-head  # This is expected to be built locally by `melange`.

From 7e0c93fe53fe6d4780c0f97906aa4a78d47b7842 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 17:09:32 +0000
Subject: [PATCH 59/72] add read permission

---
 .github/actions/build-push-kotsadm-image/action.yml | 1 +
 deploy/apko.yaml.tmpl                               | 1 -
 deploy/melange.yaml.tmpl                            | 1 +
 3 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 62b3f62fb0..6b5b730d1c 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -64,6 +64,7 @@ runs:
         mkdir -p ./packages/x86_64
         ln -s /tmp/gcsfuse/apk-repo/x86_64/*.apk ./packages/x86_64/
         ln -s /tmp/gcsfuse/apk-repo/chainguard-enterprise.rsa.pub ./packages/
+        chmod +r ./packages/chainguard-enterprise.rsa.pub
 
         # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
         cp /tmp/gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/
diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl
index 1b624bd1ac..276ab70a76 100644
--- a/deploy/apko.yaml.tmpl
+++ b/deploy/apko.yaml.tmpl
@@ -4,7 +4,6 @@ contents:
     - ./packages/
   keyring:
     - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
-    - ./packages/chainguard-enterprise.rsa.pub
     - ./melange.rsa.pub
   packages:
     - kotsadm-head  # This is expected to be built locally by `melange`.
diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index f86388720f..8f38f6553a 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -12,6 +12,7 @@ environment:
       - https://packages.wolfi.dev/os
     keyring:
       - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+      - ./packages/chainguard-enterprise.rsa.pub
     packages:
       - ca-certificates-bundle
       - busybox

From 04e44986b01f5625e26fc43cd6eef241ebe7c40c Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 17:11:29 +0000
Subject: [PATCH 60/72] can't change readonly filesystem files

---
 .github/actions/build-push-kotsadm-image/action.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 6b5b730d1c..62b3f62fb0 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -64,7 +64,6 @@ runs:
         mkdir -p ./packages/x86_64
         ln -s /tmp/gcsfuse/apk-repo/x86_64/*.apk ./packages/x86_64/
         ln -s /tmp/gcsfuse/apk-repo/chainguard-enterprise.rsa.pub ./packages/
-        chmod +r ./packages/chainguard-enterprise.rsa.pub
 
         # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
         cp /tmp/gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/

From 95c1f3e7d2fd2bf8a05724eeecf8bf4a08f3e277 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 18:47:51 +0000
Subject: [PATCH 61/72] chown runner user

---
 .github/actions/build-push-kotsadm-image/action.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 62b3f62fb0..788c7200aa 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -68,6 +68,9 @@ runs:
         # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
         cp /tmp/gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/
 
+        # Set the owner of the packages dir to the runner user
+        sudo chown -R runner:runner ./packages/
+
         ls -lR ./packages/
 
     - name: template melange and apko configs

From 71077d4137c497a4bdce893cefb7a70044ce5703 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 18:55:29 +0000
Subject: [PATCH 62/72] one more try

---
 deploy/apko.yaml.tmpl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl
index 276ab70a76..1b624bd1ac 100644
--- a/deploy/apko.yaml.tmpl
+++ b/deploy/apko.yaml.tmpl
@@ -4,6 +4,7 @@ contents:
     - ./packages/
   keyring:
     - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+    - ./packages/chainguard-enterprise.rsa.pub
     - ./melange.rsa.pub
   packages:
     - kotsadm-head  # This is expected to be built locally by `melange`.

From b2e39424c5f52323f23ec13c47ff643196a8aeef Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 19:39:20 +0000
Subject: [PATCH 63/72] print user id/group

---
 .github/actions/build-push-kotsadm-image/action.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 788c7200aa..21b80bbdf5 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -48,6 +48,10 @@ runs:
         BUCKET: replicated-apk-registry
       shell: bash
       run: |
+        whoami
+        id -u
+        id -g
+
         # Install gcsfuse
         export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
         echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list

From c1a42248850002e2dfa262e97f3f940bf7d4a911 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 20:14:21 +0000
Subject: [PATCH 64/72] one more test

---
 .github/actions/build-push-kotsadm-image/action.yml | 12 +++++-------
 .github/workflows/release.yaml                      |  1 +
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 21b80bbdf5..f4e568d7b6 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -1,6 +1,10 @@
 name: 'Build and push kotsadm image'
 description: 'Composite action for building and pushing kotsadm image'
 inputs:
+  workspace:
+    description: 'Github workspace'
+    required: true
+
   chainguard-gcp-wif-pool:
     description: 'GCP workload identity pool for Chainguard'
     required: true
@@ -48,10 +52,6 @@ runs:
         BUCKET: replicated-apk-registry
       shell: bash
       run: |
-        whoami
-        id -u
-        id -g
-
         # Install gcsfuse
         export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
         echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
@@ -72,9 +72,6 @@ runs:
         # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
         cp /tmp/gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/
 
-        # Set the owner of the packages dir to the runner user
-        sudo chown -R runner:runner ./packages/
-
         ls -lR ./packages/
 
     - name: template melange and apko configs
@@ -94,6 +91,7 @@ runs:
         archs: x86_64
         sign-with-temporary-key: true
         cache-dir: ${{ steps.cache-dir.outputs.cache_dir }}
+        keyring-append: ${{ inputs.workspace }}/packages/chainguard-enterprise.rsa.pub
 
     - uses: chainguard-images/actions/apko-publish@main
       with:
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 6507b453ea..0fef0c5808 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -190,6 +190,7 @@ jobs:
       uses: actions/checkout@v3
     - uses: ./.github/actions/build-push-kotsadm-image
       with:
+        workspace: ${{ github.workspace }}
         chainguard-gcp-wif-pool: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }}
         chainguard-gcp-sa: ${{ secrets.CHAINGUARD_GCP_SA }}
         chainguard-gcp-project-id: ${{ secrets.CHAINGUARD_GCP_PROJECT_ID }}

From e09cf09cbd9edb2e99c59e0d710abcc85abf9211 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 20:22:55 +0000
Subject: [PATCH 65/72] one more

---
 deploy/apko.yaml.tmpl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl
index 1b624bd1ac..276ab70a76 100644
--- a/deploy/apko.yaml.tmpl
+++ b/deploy/apko.yaml.tmpl
@@ -4,7 +4,6 @@ contents:
     - ./packages/
   keyring:
     - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
-    - ./packages/chainguard-enterprise.rsa.pub
     - ./melange.rsa.pub
   packages:
     - kotsadm-head  # This is expected to be built locally by `melange`.

From 0e0e29ebfec64feb1c12db2188a5e84a8ed87d01 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 20:29:59 +0000
Subject: [PATCH 66/72] don't use gcsfuse

---
 .../build-push-kotsadm-image/action.yml       | 24 +++----------------
 deploy/apko.yaml.tmpl                         |  1 +
 2 files changed, 4 insertions(+), 21 deletions(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index f4e568d7b6..6abb935333 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -47,31 +47,13 @@ runs:
       with:
         project_id: ${{ inputs.chainguard-gcp-project-id }}
 
-    - name: setup packages gcsfuse
+    - name: setup packages
       env:
         BUCKET: replicated-apk-registry
       shell: bash
       run: |
-        # Install gcsfuse
-        export GCSFUSE_REPO=gcsfuse-`lsb_release -c -s`
-        echo "deb [signed-by=/usr/share/keyrings/cloud.google.asc] https://packages.cloud.google.com/apt $GCSFUSE_REPO main" | sudo tee /etc/apt/sources.list.d/gcsfuse.list
-        curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo tee /usr/share/keyrings/cloud.google.asc
-        sudo apt-get update -y
-        sudo apt-get install gcsfuse -y
-
-        # Set up a gcsfuse RO mount to the bucket containing private packages. This is a cheap and
-        # cheerful way to get access to objects we need, without having to fetch all of them.
-        mkdir -p /tmp/gcsfuse/apk-repo
-        gcsfuse -o ro --implicit-dirs --only-dir os ${BUCKET} /tmp/gcsfuse/apk-repo
-
-        # Symlink the gcsfuse mount to ./packages/$arch/*.apk
-        mkdir -p ./packages/x86_64
-        ln -s /tmp/gcsfuse/apk-repo/x86_64/*.apk ./packages/x86_64/
-        ln -s /tmp/gcsfuse/apk-repo/chainguard-enterprise.rsa.pub ./packages/
-
-        # Make a copy of the APKINDEX.* since we'll need to write to it on package builds
-        cp /tmp/gcsfuse/apk-repo/x86_64/APKINDEX.* ./packages/x86_64/
-
+        mkdir ./packages/
+        gsutil -m cp -R gs://replicated-apk-registry/os/ ./packages/
         ls -lR ./packages/
 
     - name: template melange and apko configs
diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl
index 276ab70a76..1b624bd1ac 100644
--- a/deploy/apko.yaml.tmpl
+++ b/deploy/apko.yaml.tmpl
@@ -4,6 +4,7 @@ contents:
     - ./packages/
   keyring:
     - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+    - ./packages/chainguard-enterprise.rsa.pub
     - ./melange.rsa.pub
   packages:
     - kotsadm-head  # This is expected to be built locally by `melange`.

From 1217869e056743710e21a3d7bd3e3496372dbf0b Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 20:39:32 +0000
Subject: [PATCH 67/72] one more

---
 deploy/apko.yaml.tmpl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl
index 1b624bd1ac..276ab70a76 100644
--- a/deploy/apko.yaml.tmpl
+++ b/deploy/apko.yaml.tmpl
@@ -4,7 +4,6 @@ contents:
     - ./packages/
   keyring:
     - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
-    - ./packages/chainguard-enterprise.rsa.pub
     - ./melange.rsa.pub
   packages:
     - kotsadm-head  # This is expected to be built locally by `melange`.

From 8b3922cc63e506cfaf2293c1fe8dcb08a4ecf985 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 20:43:42 +0000
Subject: [PATCH 68/72] update

---
 .github/actions/build-push-kotsadm-image/action.yml | 5 -----
 deploy/apko.yaml.tmpl                               | 1 +
 deploy/melange.yaml.tmpl                            | 1 -
 3 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 6abb935333..9baaa63ca4 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -1,10 +1,6 @@
 name: 'Build and push kotsadm image'
 description: 'Composite action for building and pushing kotsadm image'
 inputs:
-  workspace:
-    description: 'Github workspace'
-    required: true
-
   chainguard-gcp-wif-pool:
     description: 'GCP workload identity pool for Chainguard'
     required: true
@@ -73,7 +69,6 @@ runs:
         archs: x86_64
         sign-with-temporary-key: true
         cache-dir: ${{ steps.cache-dir.outputs.cache_dir }}
-        keyring-append: ${{ inputs.workspace }}/packages/chainguard-enterprise.rsa.pub
 
     - uses: chainguard-images/actions/apko-publish@main
       with:
diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl
index 276ab70a76..1b624bd1ac 100644
--- a/deploy/apko.yaml.tmpl
+++ b/deploy/apko.yaml.tmpl
@@ -4,6 +4,7 @@ contents:
     - ./packages/
   keyring:
     - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+    - ./packages/chainguard-enterprise.rsa.pub
     - ./melange.rsa.pub
   packages:
     - kotsadm-head  # This is expected to be built locally by `melange`.
diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index 8f38f6553a..f86388720f 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -12,7 +12,6 @@ environment:
       - https://packages.wolfi.dev/os
     keyring:
       - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
-      - ./packages/chainguard-enterprise.rsa.pub
     packages:
       - ca-certificates-bundle
       - busybox

From 8fe6b466aa7013cdbfc3fea1d2df555a810234c4 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Wed, 29 Nov 2023 20:54:16 +0000
Subject: [PATCH 69/72] one more time

---
 .github/workflows/release.yaml | 1 -
 deploy/apko.yaml.tmpl          | 1 -
 2 files changed, 2 deletions(-)

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 0fef0c5808..6507b453ea 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -190,7 +190,6 @@ jobs:
       uses: actions/checkout@v3
     - uses: ./.github/actions/build-push-kotsadm-image
       with:
-        workspace: ${{ github.workspace }}
         chainguard-gcp-wif-pool: ${{ secrets.CHAINGUARD_GCP_WIF_POOL }}
         chainguard-gcp-sa: ${{ secrets.CHAINGUARD_GCP_SA }}
         chainguard-gcp-project-id: ${{ secrets.CHAINGUARD_GCP_PROJECT_ID }}
diff --git a/deploy/apko.yaml.tmpl b/deploy/apko.yaml.tmpl
index 1b624bd1ac..276ab70a76 100644
--- a/deploy/apko.yaml.tmpl
+++ b/deploy/apko.yaml.tmpl
@@ -4,7 +4,6 @@ contents:
     - ./packages/
   keyring:
     - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
-    - ./packages/chainguard-enterprise.rsa.pub
     - ./melange.rsa.pub
   packages:
     - kotsadm-head  # This is expected to be built locally by `melange`.

From 1ef2fff9f047dc4fa9d7b7ec16be2ac74afcb43a Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Thu, 30 Nov 2023 04:11:37 +0000
Subject: [PATCH 70/72] add *

---
 .github/actions/build-push-kotsadm-image/action.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/build-push-kotsadm-image/action.yml b/.github/actions/build-push-kotsadm-image/action.yml
index 9baaa63ca4..bec83c14dc 100644
--- a/.github/actions/build-push-kotsadm-image/action.yml
+++ b/.github/actions/build-push-kotsadm-image/action.yml
@@ -49,7 +49,7 @@ runs:
       shell: bash
       run: |
         mkdir ./packages/
-        gsutil -m cp -R gs://replicated-apk-registry/os/ ./packages/
+        gsutil -m cp -R gs://$BUCKET/os/* ./packages/
         ls -lR ./packages/
 
     - name: template melange and apko configs

From 4d88f3d1b5c7e155aef4fd7b9644e6f87fa499f7 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Thu, 30 Nov 2023 04:17:13 +0000
Subject: [PATCH 71/72] add chainguard-enterprise.rsa.pub to melange keyring

---
 deploy/melange.yaml.tmpl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/deploy/melange.yaml.tmpl b/deploy/melange.yaml.tmpl
index f86388720f..8f38f6553a 100644
--- a/deploy/melange.yaml.tmpl
+++ b/deploy/melange.yaml.tmpl
@@ -12,6 +12,7 @@ environment:
       - https://packages.wolfi.dev/os
     keyring:
       - https://packages.wolfi.dev/os/wolfi-signing.rsa.pub
+      - ./packages/chainguard-enterprise.rsa.pub
     packages:
       - ca-certificates-bundle
       - busybox

From a1bd1f4833bb447dbaef818576ec878d4a6c4095 Mon Sep 17 00:00:00 2001
From: Salah Aldeen Al Saleh <sg.alsaleh@gmail.com>
Date: Thu, 30 Nov 2023 14:00:46 +0000
Subject: [PATCH 72/72] don't run regression tests on the pr

---
 .github/workflows/release.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 6507b453ea..b387dc6686 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -6,7 +6,6 @@ on:
       - "v*.*.*"
     branches:
       - main
-      - integration-chainguard
 
 jobs:
   generate-tag: