Auto generated from templates by gromit

.github/dependabot.yml

@@ -1,6 +1,6 @@
 
 # Generated by: gromit policy
-# Generated on: Wed Jul 26 08:25:03 UTC 2023
+# Generated on: Thu Oct 26 05:43:34 UTC 2023
 
 version: 2
 updates:

.github/workflows/release.yml

@@ -1,12 +1,11 @@
 # Generated by: gromit policy
-# Generated on: Wed Jul 26 08:25:03 UTC 2023
+# Generated on: Thu Oct 26 05:43:34 UTC 2023
 
 # Distribution channels covered by this workflow
 # - Ubuntu and Debian
 # - RHEL and AL
 # - docker hub
 # - devenv ECR
-# - AWS mktplace (not active atm)
 # - Cloudsmith
 
 name: Release
@@ -29,6 +28,9 @@ jobs:
     name: '${{ matrix.golang_cross }}'
     runs-on: ubuntu-latest
     container: 'tykio/golang-cross:${{ matrix.golang_cross }}'
+    permissions:
+      id-token: write   # AWS OIDC JWT
+      contents: read    # actions/checkout
     strategy:
       fail-fast: false
       matrix:
@@ -37,9 +39,9 @@ jobs:
           - golang_cross: 1.19-bullseye
             goreleaser: 'ci/goreleaser/goreleaser.yml' 
             rpmvers: 'el/7 el/8 el/9 amazon/2 amazon/2023'
-            debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye'
+            debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye debian/bookworm'
     outputs:
-      tag: ${{ steps.targets.outputs.tag }}
+      tags: ${{ steps.metadata.outputs.tags }}
 
     steps:
       - name: Fix private module deps
@@ -74,32 +76,24 @@ jobs:
           username: ${{ secrets.CLOUDSMITH_USERNAME }}
           password: ${{ secrets.CLOUDSMITH_API_KEY }}
 
-      - name: Unlock agent and set tag
-        id: targets
-        shell: bash
+      - name: Unlock agent
         env:
           NFPM_STD_PASSPHRASE: ${{ secrets.SIGNING_KEY_PASSPHRASE }}
           GPG_FINGERPRINT: 12B5D62C28F57592D1575BD51ED14C59E37DAC20
           PKG_SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
-        run: |
+        run:
           ci/bin/unlock-agent.sh
-          current_tag=${GITHUB_REF##*/}
-          echo "tag=${current_tag}" >> $GITHUB_OUTPUT
 
-      - name: Delete old release assets
-        if: startsWith(github.ref, 'refs/tags')
-        uses: mknejp/delete-release-assets@v1
+
+
+      - uses: actions/cache@v3
         with:
-          token: ${{ github.token }}
-          tag: ${{ github.ref }}
-          fail-if-no-assets: false
-          fail-if-no-release: false
-          assets: |
-            *.deb
-            *.rpm
-            *.tar.gz
-            *.txt.sig
-            *.txt
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
 
       - uses: goreleaser/goreleaser-action@v4
         with:
@@ -116,6 +110,48 @@ jobs:
           DEBVERS: ${{ matrix.debvers }}
           RPMVERS: ${{ matrix.rpmvers }}
           PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+      - uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
+          role-session-name: cipush
+          aws-region: eu-central-1
+          # Don't mask to pass it across job boundaries
+          mask-aws-account-id: false
+
+      - uses: aws-actions/amazon-ecr-login@v1
+        id: ecr
+        with:
+          mask-password: 'true'
+
+      - name: Docker metadata for CI
+        id: metadata
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ steps.ecr.outputs.registry }}/tyk-pump
+          flavor: |
+            latest=false
+          tags: |
+            type=ref,event=branch
+            type=ref,event=pr
+            type=sha,format=long
+            type=semver,pattern=v{{major}}.{{minor}},prefix=v
+            type=semver,pattern=v{{version}},prefix=v
+
+      - name: CI push
+        shell: bash
+        env:
+          t: ${{ steps.metadata.outputs.tags }}
+          build_tag: ${{ startswith(github.ref, 'refs/tags') && github.ref_name || 'v0.0.0' }}
+        run: |
+          set +e
+          IFS=$'\n' tags=($t)
+          for tag in "${tags[@]}"; do
+             for arch in amd64 arm64; do
+                 docker tag tykio/tyk-pump-docker-pub:${build_tag}-${arch} ${tag}-${arch} && docker push ${tag}-${arch}
+             done
+             docker manifest create ${tag} ${tag}-amd64 ${tag}-arm64 && docker manifest push ${tag}
+          done
+
       - uses: actions/upload-artifact@v3
         with:
           name: deb
@@ -131,73 +167,102 @@ jobs:
           path: |
             dist/*.rpm
             !dist/*PAYG*.rpm
-  ci:
-    needs:
-      - goreleaser
+  api-tests:
+    needs: goreleaser
     runs-on: ubuntu-latest
     permissions:
       id-token: write   # This is required for requesting the JWT
       contents: read    # This is required for actions/checkout
+    strategy:
+      fail-fast: false
+      matrix:
+        conf: [ "sha256", "murmur64" ]
+        db: [ "mongo44", "postgres15" ]
+        include:
+          - db: postgres15
+            markers: "and not sql"
 
     steps:
-      - name: Shallow checkout of tyk-pump
-        uses: actions/checkout@v3
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
+      - name: Checkout of tyk-automated-tests
+        uses: actions/checkout@v3
+        with:
+          repository: TykTechnologies/tyk-automated-tests
+          token: ${{ secrets.ORG_GH_TOKEN }}
+          ref: ${{ startsWith(github.ref_name, 'release-') && github.ref_name || 'master' }}
+          path: tyk-automated-tests
+
       - uses: aws-actions/configure-aws-credentials@v2
         with:
           role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
           role-session-name: cipush
           aws-region: eu-central-1
 
-      - name: Login to Amazon ECR
-        id: login-ecr
+      - id: ecr
         uses: aws-actions/amazon-ecr-login@v1
-
-      - uses: actions/download-artifact@v3
         with:
-          name: deb
+          mask-password: 'true'
 
-      - name: Docker metadata
-        id: metadata
-        uses: docker/metadata-action@v4
+        # Only ${{ github.actor }} has access
+        # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys
+      - name: Setup tmate session only in debug mode
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ runner.debug == '1' }}
         with:
-          images: ${{ steps.login-ecr.outputs.registry }}/tyk-pump
-          flavor: |
-            latest=false
-            prefix=v
-          tags: |
-              type=semver,pattern=v{{version}}
-              type=semver,pattern=v{{major}}.{{minor}}
-              type=semver,pattern=v{{major}}
-              type=ref,event=branch
-
-      - uses: docker/setup-qemu-action@v2
+          detached: true
+          limit-access-to-actor: true
 
-      - uses: docker/setup-buildx-action@v2
-
-      - name: CI build
-        uses: docker/build-push-action@v4
-        with:
-          push: true
-          context: "."
-          file: ci/Dockerfile.std
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            ${{ steps.metadata.outputs.tags }}
-            ${{ steps.login-ecr.outputs.registry }}/tyk-pump:${{ needs.goreleaser.outputs.tag }}
-            ${{ steps.login-ecr.outputs.registry }}/tyk-pump:${{ github.sha }}
+      - name: env up
+        shell: bash
+        working-directory: tyk-automated-tests/ci/auto
+        id: env_up
+        env:
+          t: ${{ needs.goreleaser.outputs.tags }}
+          pull_policy: 'if_not_present'
+          # gw and dash use the same branch names
+          gw_dash_image_tag: ${{ ( (github.repository == 'TykTechnologies/tyk' || github.repository == 'TykTechnologies/tyk-analytics') && startsWith(github.ref_name, 'release-') ) && github.ref_name || 'master' }}
+          pump_image_tag: ${{ ( github.repository == 'TykTechnologies/tyk-pump' && startsWith(github.ref_name, 'release-') ) && github.ref_name || 'master' }}
+          sink_image_tag: ${{ ( github.repository == 'TykTechnologies/tyk-sink' && startsWith(github.ref_name, 'release-') ) && github.ref_name || 'master' }}
+          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
+          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
+          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
+        run: |
+          echo CI tags: $t
+          tags=($t)
+          echo First tag: ${tags[0]}
+          # Start customising the env
+          echo "registry=${{ steps.ecr.outputs.registry }}
+          tyk_image=\${registry}/tyk:${gw_dash_image_tag}
+          tyk_analytics_image=\${registry}/tyk-analytics:${gw_dash_image_tag}
+          tyk_sink_image=\${registry}/tyk-sink:${sink_image_tag}
+          tyk_pump_image=\${registry}/tyk-pump:${pump_image_tag}
+          # override default above with just built tag
+          tyk_image=${tags[0]}
+          tyk_alfa_image=${tags[0]}
+          tyk_beta_image=${tags[0]}
+          # base dir for config files
+          confs_dir=./pro-ha
+          # pick database to use
+          env_file=local-${{ matrix.db }}.env
+          " > versions.env
+          # Add Tyk component config variations to $env_file
+          cat confs/${{ matrix.conf }}.env >> local-${{ matrix.db }}.env
+          # bring up env, the project name is important
+          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d
+          ./dash-bootstrap.sh http://localhost:3000
+          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d
 
-  sbom:
-    needs: ci
-    uses: TykTechnologies/github-actions/.github/workflows/sbom.yaml@main
-    secrets:
-      TF_API_TOKEN: ${{ secrets.TF_API_TOKEN }}
-      DEPDASH_URL: ${{ secrets.DEPDASH_URL }}
-      DEPDASH_KEY: ${{ secrets.DEPDASH_KEY }}
-      ORG_GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
+      - name: Run tests
+        working-directory: tyk-automated-tests/ci/auto
+        run: |
+          docker run --rm --network auto_default --env-file pytest.env \
+          ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:${{ startsWith(github.ref_name, 'release-') && github.ref_name || 'master' }} \
+          pytest -c pytest_ci.ini --ci -m "not local and not dind ${{ matrix.markers }}"
 
+# TODO: PR comment, using just one comment and updating, not a new comment each time
 
   upgrade-deb:
     if: startsWith(github.ref, 'refs/tags')
@@ -213,10 +278,11 @@ jobs:
           - amd64
           - arm64
         distro:
-          - ubuntu:jammy
           - ubuntu:bionic
           - ubuntu:focal
+          - ubuntu:jammy
           - debian:bullseye
+          - debian:bookworm
 
     steps:
       - uses: actions/checkout@v3
@@ -260,8 +326,9 @@ jobs:
       fail-fast: false
       matrix:
         distro:
-          - ubi9/ubi
-          - ubi8/ubi
+          - amazonlinux:2023
+          - registry.access.redhat.com/ubi8/ubi
+          - registry.access.redhat.com/ubi9/ubi
 
     steps:
       - uses: actions/checkout@v3
@@ -276,7 +343,7 @@ jobs:
 
       - name: generate dockerfile
         run: |
-          echo 'FROM registry.access.redhat.com/${{ matrix.distro }}
+          echo 'FROM ${{ matrix.distro }}
           COPY tyk-pump*.x86_64.rpm /tyk-pump.rpm
           RUN yum install --allowerasing -y curl
           RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.rpm.sh | bash && yum install -y tyk-pump-1.6.0-1
@@ -293,20 +360,35 @@ jobs:
           push: false
 
   smoke-tests:
-    if: startsWith(github.ref, 'refs/tags')
     needs:
       - goreleaser
+    permissions:
+      id-token: write   # This is required for requesting the JWT
+      contents: read    # This is required for actions/checkout
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 1
 
-      - name: Run tests
+      - uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
+          role-session-name: cipush
+          aws-region: eu-central-1
+
+      - id: ecr
+        uses: aws-actions/amazon-ecr-login@v1
+        with:
+          mask-password: 'true'
+
+      - name: Run ci/tests
         shell: bash
         env:
           GITHUB_TAG: ${{ github.ref }}
+          GATEWAY_IMAGE: ${{ steps.ecr.outputs.registry }}/tyk:${{ github.sha }}
+          PLUGIN_COMPILER_IMAGE: ${{ steps.ecr.outputs.registry }}/tyk-plugin-compiler:sha-${{ github.sha }}
         run: |
           set -eaxo pipefail
           if [ ! -d smoke-tests ]; then
@@ -335,3 +417,11 @@ jobs:
               fi
           done
 
+
+  sbom:
+    needs: goreleaser
+    uses: TykTechnologies/github-actions/.github/workflows/sbom.yaml@main
+    secrets:
+      DEPDASH_URL: ${{ secrets.DEPDASH_URL }}
+      DEPDASH_KEY: ${{ secrets.DEPDASH_KEY }}
+      ORG_GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}

ci/Dockerfile.std

@@ -1,18 +1,16 @@
-
 # Generated by: gromit policy
-# Generated on: Wed Jul 26 08:25:03 UTC 2023
+# Generated on: Thu Oct 26 05:43:34 UTC 2023
 
-FROM debian:bullseye-slim
+FROM debian:bookworm-slim
 ARG TARGETARCH
 
 ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update \
     && apt-get dist-upgrade -y ca-certificates
 
-
 # Remove some things to decrease CVE surface
-RUN  apt-get remove -y --allow-remove-essential --auto-remove curl libtiff5 ncurses-base \
+RUN dpkg --purge --force-remove-essential curl libtiff5 ncurses-base \
 	&& rm /usr/bin/passwd && rm /usr/sbin/adduser
 
 # Clean up caches, unwanted .a and .o files

ci/aws/byol.pkr.hcl

@@ -1,6 +1,6 @@
 
 # Generated by: gromit policy
-# Generated on: Wed Jul 26 08:25:03 UTC 2023
+# Generated on: Thu Oct 26 05:43:34 UTC 2023
 
 packer {
   required_plugins {