[SYSE-288] tyk-pump/master test-api templates

.github/dependabot.yml

@@ -1,6 +1,6 @@
 
 # Generated by: gromit policy
-# Generated on: Fri Oct 27 04:05:19 UTC 2023
+# Generated on: Tue Nov 21 18:30:13 UTC 2023
 
 version: 2
 updates:

.github/workflows/release.yml

@@ -1,5 +1,5 @@
 # Generated by: gromit policy
-# Generated on: Fri Oct 27 04:05:19 UTC 2023
+# Generated on: Tue Nov 21 18:30:13 UTC 2023
 
 # Distribution channels covered by this workflow
 # - Ubuntu and Debian
@@ -52,26 +52,26 @@ jobs:
           git config --global url."https://${TOKEN}@github.com".insteadOf "https://github.com"
 
       - name: Checkout of tyk-pump
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - name: "Add Git safe.directory"
         run: git config --global --add safe.directory $GITHUB_WORKSPACE
 
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
 
-      - uses: docker/setup-buildx-action@v2
+      - uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
         if: startsWith(github.ref, 'refs/tags')
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Login to Cloudsmith
         if: startsWith(github.ref, 'refs/tags')
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: docker.tyk.io
           username: ${{ secrets.CLOUDSMITH_USERNAME }}
@@ -111,7 +111,7 @@ jobs:
           DEBVERS: ${{ matrix.debvers }}
           RPMVERS: ${{ matrix.rpmvers }}
           PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
-      - uses: aws-actions/configure-aws-credentials@v2
+      - uses: aws-actions/configure-aws-credentials@v4
         with:
           role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
           role-session-name: cipush
@@ -121,12 +121,14 @@ jobs:
 
       - uses: aws-actions/amazon-ecr-login@v1
         id: ecr
+        if: ${{ matrix.golang_cross == '1.19-bullseye' }}
         with:
           mask-password: 'true'
 
       - name: Docker metadata for CI
         id: metadata
-        uses: docker/metadata-action@v4
+        if: ${{ matrix.golang_cross == '1.19-bullseye' }}
+        uses: docker/metadata-action@v5
         with:
           images: ${{ steps.ecr.outputs.registry }}/tyk-pump
           flavor: |
@@ -139,6 +141,7 @@ jobs:
             type=semver,pattern=v{{version}},prefix=v
 
       - name: CI push
+        if: ${{ matrix.golang_cross == '1.19-bullseye' }}
         shell: bash
         env:
           t: ${{ steps.metadata.outputs.tags }}
@@ -154,6 +157,7 @@ jobs:
           done
 
       - uses: actions/upload-artifact@v3
+        if: ${{ matrix.golang_cross == '1.19-bullseye' }}
         with:
           name: deb
           retention-days: 1
@@ -162,14 +166,157 @@ jobs:
             !dist/*PAYG*.deb
 
       - uses: actions/upload-artifact@v3
+        if: ${{ matrix.golang_cross == '1.19-bullseye' }}
         with:
           name: rpm
           retention-days: 1
           path: |
             dist/*.rpm
-            !dist/*PAYG*.rpm 
+            !dist/*PAYG*.rpm
+  api-tests:
+    needs: goreleaser
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write   # This is required for requesting the JWT
+      contents: read    # This is required for actions/checkout
+    strategy:
+      fail-fast: false
+      matrix:
+        conf: [ "sha256", "murmur64" ]
+        db: [ "mongo44", "postgres15" ]
+        include:
+          - db: postgres15
+            markers: "and not sql"
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
+          role-session-name: cipush
+          aws-region: eu-central-1
+
+      - id: ecr
+        uses: aws-actions/amazon-ecr-login@v1
+        with:
+          mask-password: 'true'
+          mask-aws-account-id: 'true'
+
+        # Only ${{ github.actor }} has access
+        # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys
+
+      - name: Setup tmate session only in debug mode
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ runner.debug == '1' }}
+        with:
+          detached: true
+          limit-access-to-actor: true
+
+      - name: fetch env from tyk-ci
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release download v1.2 --repo github.com/TykTechnologies/tyk-ci -O env.tgz
+          tar xzvf env.tgz
+
+      - name: env up
+        shell: bash
+        working-directory: auto
+        id: env_up
+        env:
+          t: ${{ needs.goreleaser.outputs.tags }}
+          pull_policy: 'if_not_present'
+          # default tag to use
+          default_image_tag: ${{ startsWith(github.ref_name, 'release-') && github.ref_name || 'master' }}
+          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
+          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
+          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
+        run: |
+          echo CI tags: $t
+          tags=($t)
+          echo First tag: ${tags[0]}
+          # Start customising the env
+          echo "registry=${{ steps.ecr.outputs.registry }}
+          tyk_image=\${registry}/tyk:${default_image_tag}
+          tyk_analytics_image=\${registry}/tyk-analytics:${default_image_tag}
+          tyk_sink_image=\${registry}/tyk-sink:${default_image_tag}
+          tyk_pump_image=\${registry}/tyk-pump:${default_image_tag}
+          # override default above with just built tag
+          tyk_pump_image=${tags[0]}
+          # alfa and beta have to come after the override
+          tyk_alfa_image=\${tyk_image}
+          tyk_beta_image=\${tyk_image}
+          # base dir for config files
+          confs_dir=./pro-ha
+          # pick database to use
+          env_file=local-${{ matrix.db }}.env
+          " > versions.env
+          # Add Tyk component config variations to $env_file
+          cat confs/${{ matrix.conf }}.env >> local-${{ matrix.db }}.env
+          # bring up env, the project name is important
+          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d
+          ./dash-bootstrap.sh http://localhost:3000
+          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d
+
+      - name: Run tests
+        working-directory: auto
+        id: test_execution
+        run: |
+          echo "## API tests ${{ matrix.db }} ${{ matrix.conf }}" >> $GITHUB_STEP_SUMMARY
+          if docker run --rm --network auto_default --env-file pytest.env -v ${{ github.workspace }}/reports:/app/reports \
+             ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:${{ startsWith(github.ref_name, 'release-') && github.ref_name || 'master' }} \
+             pytest -c pytest_ci.ini --ci -m "not local and not dind ${{ matrix.markers }}" > tests.out; then
+               echo "All tests passed!" >> $GITHUB_STEP_SUMMARY
+          else
+               echo "::error title=API tests ${{ matrix.db }} ${{ matrix.conf }}::Test execution failed"
+               cat tests.out >> $GITHUB_STEP_SUMMARY
+               docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env logs tyk tyk-analytics
+               exit 1
+          fi
+
+      - name: Archive Integration tests report
+        if: ${{ always() }} 
+        uses: actions/upload-artifact@v3
+        with:
+            name: api-test-report
+            retention-days: 3
+            path: ${{ github.workspace }}/reports
+
+      - name: Fetch commit author
+        if: ${{ steps.test_execution.outcome != 'success' && !github.event.pull_request.draft }}            
+        run: echo "GIT_USER_EMAIL=$(git show -s --format='%ae' HEAD)" >> $GITHUB_ENV
+
+      - name: Fetch slack user
+        if: ${{ steps.test_execution.outcome != 'success' && !github.event.pull_request.draft }}   
+        id: fetch_slack_user
+        uses: TykTechnologies/github-actions/.github/actions/github-to-slack@main
+        with:
+          github_email: ${{ env.GIT_USER_EMAIL }}
+
+      - name: Notify slack
+        if: ${{ steps.test_execution.outcome != 'success' && !github.event.pull_request.draft }}
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_WEBHOOK: ${{ secrets.API_TEST_ALERT_SLACK_WEBHOOK }}
+          SLACK_COLOR: ${{ job.status }}
+          SLACK_TITLE: "Result: ${{ steps.test_execution.outcome }}"
+          SLACK_USERNAME: API INTEGRATION TESTS
+          SLACK_MESSAGE: "*Test*: ${{ matrix.db }}-${{ matrix.conf }}, *Author*: ${{ steps.fetch_slack_user.outputs.slack-user-name }}"
+          SLACK_FOOTER: "<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|SEE EXECUTION DETAILS HERE>"
+
+      - name: Comment on PR
+        if: ${{ always() && !github.event.pull_request.draft }}
+        uses: mshick/add-pr-comment@v2
+        with:
+          message: |
+            **API tests result - ${{ matrix.db }}-${{ matrix.conf }} env: ${{ steps.test_execution.outcome }}** ${{ env.STATUS }} 
+            Branch used: ${{ github.ref }}
+            Commit: ${{ github.event.after }} ${{ github.event.commits[0].message }}
+            Triggered by: ${{ github.event_name }} (@${{ github.actor }})
+            [Execution page](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})
+          repo-token: ${{ secrets.ORG_GH_TOKEN }}
+          message-id: ${{ matrix.db }}-${{ matrix.conf }}
+        env:
+          STATUS: "${{ steps.test_execution.outcome == 'success' && ':white_check_mark:' || ':no_entry_sign:' }}"
 
-# TODO: PR comment, using just one comment and updating, not a new comment each time
 
   upgrade-deb:
     if: startsWith(github.ref, 'refs/tags')
@@ -192,17 +339,17 @@ jobs:
           - debian:bookworm
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - uses: actions/download-artifact@v3
         with:
           name: deb
 
-      - uses: docker/setup-qemu-action@v2
+      - uses: docker/setup-qemu-action@v3
 
-      - uses: docker/setup-buildx-action@v2
+      - uses: docker/setup-buildx-action@v3
 
       - name: generate dockerfile
         run: |
@@ -215,7 +362,7 @@ jobs:
           ' > Dockerfile
 
       - name: install on ${{ matrix.distro }}
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: "."
           platforms: linux/${{ matrix.arch }}
@@ -238,15 +385,15 @@ jobs:
           - registry.access.redhat.com/ubi9/ubi
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
       - uses: actions/download-artifact@v3
         with:
           name: rpm
 
-      - uses: docker/setup-buildx-action@v2
+      - uses: docker/setup-buildx-action@v3
 
       - name: generate dockerfile
         run: |
@@ -260,12 +407,11 @@ jobs:
           ' > Dockerfile
 
       - name: install on ${{ matrix.distro }}
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: "."
           file: Dockerfile
           push: false
-
   smoke-tests:
     needs:
       - goreleaser
@@ -275,7 +421,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
@@ -285,15 +431,15 @@ jobs:
           role-session-name: cipush
           aws-region: eu-central-1
 
-      - uses: aws-actions/amazon-ecr-login@v1
-        id: ecr
+      - id: ecr
+        uses: aws-actions/amazon-ecr-login@v1
         with:
           mask-password: 'true'
-          
+
       - name: Run ci/tests
         shell: bash
         env:        
-          GITHUB_TAG: ${{ github.ref }}
+          GITHUB_TAG: ${{ github.ref }} /* mdcb or dash */
         run: |
           set -eaxo pipefail
           if [ ! -d smoke-tests ]; then

ci/Dockerfile.std

@@ -1,5 +1,5 @@
 # Generated by: gromit policy
-# Generated on: Fri Oct 27 04:05:19 UTC 2023
+# Generated on: Tue Nov 21 18:30:13 UTC 2023
 
 FROM debian:bullseye-slim
 ARG TARGETARCH
@@ -18,8 +18,7 @@ RUN rm -rf /root/.cache \
     && apt-get -y autoremove \
     && apt-get clean \
     && rm -rf /usr/include/* \
-    && find /usr/lib -type f -name '*.a' -delete \
-    && find /usr/lib -type f -name '*.o' -delete
+    && find /usr/lib -type f -name '*.a' -o -name '*.o' -delete
 
 # Comment this to test in dev
 COPY *${TARGETARCH}.deb /
@@ -34,5 +33,4 @@ WORKDIR /opt/tyk-pump/
 # Uncomment this to test in dev
 # COPY tyk-pump .
 ENTRYPOINT ["/opt/tyk-pump/tyk-pump" ]
-
 CMD [ "--conf=/opt/tyk-pump/pump.conf" ]

ci/aws/byol.pkr.hcl

@@ -1,6 +1,6 @@
 
 # Generated by: gromit policy
-# Generated on: Fri Oct 27 04:05:19 UTC 2023
+# Generated on: Tue Nov 21 18:30:13 UTC 2023
 
 packer {
   required_plugins {

ci/goreleaser/goreleaser.yml

@@ -1,5 +1,5 @@
 # Generated by: gromit policy
-# Generated on: Fri Oct 27 04:05:19 UTC 2023
+# Generated on: Tue Nov 21 18:30:13 UTC 2023
 
 # Check the documentation at http://goreleaser.com
 # This project needs CGO_ENABLED=1 and the cross-compiler toolchains for

ci/install/before_install.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Generated by: gromit policy
-# Generated on: Fri Oct 27 04:05:19 UTC 2023
+# Generated on: Tue Nov 21 18:30:13 UTC 2023
 
 echo "Creating user and group..."
 GROUPNAME="tyk"

ci/install/post_install.sh

@@ -2,7 +2,7 @@
 
 
 # Generated by: gromit policy
-# Generated on: Fri Oct 27 04:05:19 UTC 2023
+# Generated on: Tue Nov 21 18:30:13 UTC 2023
 
 # If "True" the install directory ownership will be changed to "tyk:tyk"
 change_ownership="True"