Auto generated from templates by gromit (#800)

fixes SYSE-332: set pipefail to fail tests if anything goes wrong
delete int-image.yml
fix version package for dashboard (@titpetric)
test controller for UI tests
action version bump for artifacts, cache
Co-authored-by: Gromit <policy@gromit>

.github/workflows/release.yml

@@ -1,5 +1,5 @@
 # Generated by: gromit policy
-# Generated on: Thu Jan 18 17:24:56 UTC 2024
+# Generated on: Mon Mar 11 11:43:04 UTC 2024
 
 # Distribution channels covered by this workflow
 # - Ubuntu and Debian
@@ -42,11 +42,15 @@ jobs:
             debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye debian/bookworm'
     outputs:
       tags: ${{ steps.metadata.outputs.tags }}
+      commit_author: ${{ steps.fetch-author.outputs.commit_author}}
     steps:
       - name: Checkout of tyk-pump
         uses: actions/checkout@v4
         with:
           fetch-depth: 1
+      - name: Get commit author
+        id: fetch-author
+        run: echo "commit_author=$(git show -s --format='%ae' HEAD)" >> $GITHUB_OUTPUT
       - uses: docker/setup-qemu-action@v3
       - uses: docker/setup-buildx-action@v3
       - name: Login to DockerHub
@@ -62,7 +66,7 @@ jobs:
           registry: docker.tyk.io
           username: ${{ secrets.CLOUDSMITH_USERNAME }}
           password: ${{ secrets.CLOUDSMITH_API_KEY }}
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         with:
           path: |
             ~/.cache/go-build
@@ -140,29 +144,29 @@ jobs:
              done
              docker manifest create ${tag} ${tag}-amd64 ${tag}-arm64 && docker manifest push ${tag}
           done
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: ${{ matrix.golang_cross == '1.21-bullseye' }}
         with:
           name: deb
           retention-days: 1
           path: |
             dist/*.deb
             !dist/*PAYG*.deb
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: ${{ matrix.golang_cross == '1.21-bullseye' }}
         with:
           name: rpm
           retention-days: 1
           path: |
             dist/*.rpm
             !dist/*PAYG*.rpm
-  test-controller:
+  test-controller-api:
     needs: goreleaser
     runs-on: ubuntu-latest
-    container: tykio/gromit:v1.6
+    container: tykio/gromit:v1.7
     outputs:
-      conf: ${{ steps.params.outputs.conf }}
-      db: ${{ steps.params.outputs.db }}
+      conf: ${{ steps.params.outputs.api_conf }}
+      db: ${{ steps.params.outputs.api_db }}
       pump: ${{ steps.params.outputs.pump }}
       sink: ${{ steps.params.outputs.sink }}
       gd_tag: ${{ steps.params.outputs.gd_tag }}
@@ -177,19 +181,20 @@ jobs:
           TAGS: ${{ needs.goreleaser.outputs.tags }}
           IS_PR: ${{startsWith(github.event_name, 'pull_request') && 'yes' }}
           IS_TAG: ${{startsWith(github.ref, 'refs/tags') && 'yes' }}
+          JOB: api
         run: gromit policy controller --loglevel debug | tee -a "$GITHUB_OUTPUT"
   api-tests:
-    needs: test-controller
+    needs: [goreleaser, test-controller-api]
     runs-on: ubuntu-latest
     permissions:
       id-token: write # This is required for requesting the Github JWT
       contents: read # This is required for actions/checkout
     strategy:
       fail-fast: false
       matrix:
-        conf: ${{ fromJson(needs.test-controller.outputs.conf) }}
-        db: ${{ fromJson(needs.test-controller.outputs.db) }}
-        sink: ${{ fromJson(needs.test-controller.outputs.sink) }}
+        conf: ${{ fromJson(needs.test-controller-api.outputs.conf) }}
+        db: ${{ fromJson(needs.test-controller-api.outputs.db) }}
+        sink: ${{ fromJson(needs.test-controller-api.outputs.sink) }}
         include:
           - db: postgres15
             markers: "and not sql"
@@ -207,7 +212,7 @@ jobs:
           # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys
       - name: Setup tmate session only in debug mode
         uses: mxschmitt/action-tmate@v3
-        if: ${{ runner.debug == '1' }}
+        if: runner.debug == '1'
         with:
           detached: true
           limit-access-to-actor: true
@@ -227,41 +232,67 @@ jobs:
           TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
           TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
           ECR: ${{ steps.ecr.outputs.registry }}
-          VERSIONS: ${{ needs.test-controller.outputs.versions}}
+          VERSIONS: ${{ needs.test-controller-api.outputs.versions}}
         run: "echo \"ECR=${ECR}\n$VERSIONS\n\ntyk_sink_image=${{matrix.sink}} \n# base dir for config files\nconfs_dir=./pro-ha\n# pick database to use\nenv_file=local-${{ matrix.db }}.env\n\" > versions.env\nprintf '::group::versions\\n%s\\n::endgroup::' \"$(cat versions.env)\"\n# Add Tyk component config variations to $env_file\ncat confs/${{ matrix.conf }}.env >> local-${{ matrix.db }}.env\n# bring up env, the project name is important\ndocker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d\n./dash-bootstrap.sh http://localhost:3000\ndocker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d\n"
       - name: Run tests
         working-directory: auto
         id: test_execution
         run: |
+          set -o pipefail
           echo "### API tests ${{ matrix.db }} ${{ matrix.conf }}" >> $GITHUB_STEP_SUMMARY
           if docker run --rm --network auto_default --env-file pytest.env -v ${{ github.workspace }}/reports:/app/reports \
-             ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:${{ needs.test-controller.outputs.gd_tag }} \
+             ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:${{ needs.test-controller-api.outputs.gd_tag }} \
              pytest -c pytest_ci.ini --ci -m "not local and not dind ${{ matrix.markers }}" | tee tests.out; then
                echo "All tests passed!" >> $GITHUB_STEP_SUMMARY
           else
                echo "::error title=API tests ${{ matrix.db }} ${{ matrix.conf }}::Test execution failed"
                cat tests.out >> $GITHUB_STEP_SUMMARY
-               docker logs tyk tyk-analytics
                exit 1
           fi
+      - name: Docker logs for all components
+        if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
+        working-directory: auto
+        env:
+          pull_policy: 'if_not_present'
+          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
+          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
+          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
+          ECR: ${{ steps.ecr.outputs.registry }}
+          VERSIONS: ${{ needs.test-controller-api.outputs.versions }}
+        run: |
+          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile all logs | sort > ${{ github.workspace }}/docker-compose-api.log
+          echo "::group::DockerLogs"
+          cat ${{ github.workspace }}/docker-compose-api.log
+          echo "::endgroup::"
+      - name: Upload Artifact
+        uses: actions/upload-artifact@v4
+        if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
+        with:
+          name: docker-compose-logs-api-${{ matrix.db }}-${{ matrix.conf }}-${{ github.run_id }}
+          path: ${{ github.workspace }}/docker-compose-api.log
+          retention-days: 3
+          override: true
       - name: Archive Integration tests report
-        if: ${{ always() }}
-        uses: actions/upload-artifact@v3
+        if: always()
+        uses: actions/upload-artifact@v4
         with:
-          name: api-test-report
+          name: api-test-report-${{ matrix.db }}-${{ matrix.conf }}-${{ github.run_id }}
           retention-days: 3
           path: ${{ github.workspace }}/reports
+          override: true
       - name: Fetch commit author
-        if: ${{ steps.test_execution.outcome != 'success' && !github.event.pull_request.draft }}
-        run: echo "GIT_USER_EMAIL=$(git show -s --format='%ae' HEAD)" >> $GITHUB_ENV
+        if: failure() && steps.test_execution.outcome != 'success' && github.event_name == 'push'
+        env:
+          USER_EMAIL: ${{ needs.goreleaser.outputs.commit_author }}
+        run: echo "GIT_USER_EMAIL=$USER_EMAIL" >> $GITHUB_ENV
       - name: Fetch slack user
-        if: ${{ steps.test_execution.outcome != 'success' && !github.event.pull_request.draft }}
+        if: failure() && steps.test_execution.outcome != 'success' && github.event_name == 'push'
         id: fetch_slack_user
         uses: TykTechnologies/github-actions/.github/actions/github-to-slack@main
         with:
           github_email: ${{ env.GIT_USER_EMAIL }}
       - name: Notify slack
-        if: ${{ steps.test_execution.outcome != 'success' && !github.event.pull_request.draft }}
+        if: failure() && steps.test_execution.outcome != 'success' && github.event_name == 'push'
         uses: rtCamp/action-slack-notify@v2
         env:
           SLACK_WEBHOOK: ${{ secrets.API_TEST_ALERT_SLACK_WEBHOOK }}
@@ -270,15 +301,6 @@ jobs:
           SLACK_USERNAME: API INTEGRATION TESTS
           SLACK_MESSAGE: "*Test*: ${{ matrix.db }}-${{ matrix.conf }}, *Author*: ${{ steps.fetch_slack_user.outputs.slack-user-name }}"
           SLACK_FOOTER: "<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|SEE EXECUTION DETAILS HERE>"
-      - name: Comment on PR
-        if: ${{ always() && !github.event.pull_request.draft }}
-        uses: mshick/add-pr-comment@v2
-        with:
-          message: "**API tests result - ${{ matrix.db }}-${{ matrix.conf }} env: ${{ steps.test_execution.outcome }}** ${{ env.STATUS }} \nBranch used: ${{ github.ref }}\nCommit: ${{ github.event.after }} ${{ github.event.commits[0].message }}\nTriggered by: ${{ github.event_name }} (@${{ github.actor }})\n[Execution page](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})\n"
-          repo-token: ${{ secrets.ORG_GH_TOKEN }}
-          message-id: ${{ matrix.db }}-${{ matrix.conf }}
-        env:
-          STATUS: "${{ steps.test_execution.outcome == 'success' && ':white_check_mark:' || ':no_entry_sign:' }}"
   upgrade-deb:
     services:
       httpbin.org:
@@ -301,7 +323,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: deb
       - uses: docker/setup-qemu-action@v3
@@ -339,7 +361,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: rpm
       - uses: docker/setup-buildx-action@v3

ci/Dockerfile.std

@@ -1,5 +1,5 @@
 # Generated by: gromit policy
-# Generated on: Thu Jan 18 17:24:56 UTC 2024
+# Generated on: Mon Mar 11 11:43:04 UTC 2024
 
 FROM debian:bullseye-slim
 ARG TARGETARCH

ci/bin/pc.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Generated by: gromit policy
-# Generated on: Thu Jan 18 17:24:56 UTC 2024
+# Generated on: Mon Mar 11 11:43:04 UTC 2024
 
 # goreleaser calls a custom publisher for each artefact packagecloud
 # expects the distro version when pushing this script bridges both by

ci/bin/unlock-agent.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # Generated by: gromit policy
-# Generated on: Thu Jan 18 17:24:56 UTC 2024
+# Generated on: Mon Mar 11 11:43:04 UTC 2024
 
 # Get the GPG fingerprint with gpg --with-keygrip --list-secret-keys
 if [[ -z "${PKG_SIGNING_KEY}" || -z "${NFPM_STD_PASSPHRASE}" || -z "${GPG_FINGERPRINT}" ]]; then

ci/goreleaser/goreleaser.yml

@@ -1,5 +1,5 @@
 # Generated by: gromit policy
-# Generated on: Thu Jan 18 17:24:56 UTC 2024
+# Generated on: Mon Mar 11 11:43:04 UTC 2024
 
 # Check the documentation at http://goreleaser.com
 # This project needs CGO_ENABLED=1 and the cross-compiler toolchains for
@@ -126,10 +126,7 @@ publishers:
   - name: tyk-pump-unstable
     env:
       - PACKAGECLOUD_TOKEN={{ .Env.PACKAGECLOUD_TOKEN }}
-      - REPO=tyk/tyk-pump-unstable
-      - RPMVERS={{ .Env.RPMVERS }}
-      - DEBVERS={{ .Env.DEBVERS }}
-    cmd: ./ci/bin/pc.sh {{ .ArtifactPath }}
+    cmd: packagecloud publish --debvers "{{ .Env.DEBVERS }}" --rpmvers "{{ .Env.RPMVERS }}" tyk/tyk-pump-unstable {{ .ArtifactPath }}
 # This disables archives
 archives:
   - format: binary

ci/install/before_install.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Generated by: gromit policy
-# Generated on: Thu Jan 18 17:24:56 UTC 2024
+# Generated on: Mon Mar 11 11:43:04 UTC 2024
 
 echo "Creating user and group..."
 GROUPNAME="tyk"

ci/install/post_install.sh

@@ -2,7 +2,7 @@
 
 
 # Generated by: gromit policy
-# Generated on: Thu Jan 18 17:24:56 UTC 2024
+# Generated on: Mon Mar 11 11:43:04 UTC 2024
 
 # If "True" the install directory ownership will be changed to "tyk:tyk"
 change_ownership="True"

ci/install/post_remove.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # Generated by: gromit policy
-# Generated on: Thu Jan 18 17:24:56 UTC 2024
+# Generated on: Mon Mar 11 11:43:04 UTC 2024
 
 
 cleanRemove() {

ci/install/post_trans.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 # Generated by: gromit policy
-# Generated on: Thu Jan 18 17:24:56 UTC 2024
+# Generated on: Mon Mar 11 11:43:04 UTC 2024
 
 if command -V systemctl >/dev/null 2>&1; then
     if [ ! -f /lib/systemd/system/tyk-pump.service ]; then