release.yml

# Generated by: gromit policy

# Distribution channels covered by this workflow
# - Ubuntu and Debian
# - RHEL and AL
# - docker hub
# - devenv ECR
# - Cloudsmith

name: Release
on:
  # Trigger release every monday at midnight for master CI images
  schedule:
    - cron: "0 0 * * 1"
  pull_request:
  push:
    branches:
      - master
      - release-**
    tags:
      - 'v*'
env:
  GOPRIVATE: github.com/TykTechnologies
jobs:
  goreleaser:
    name: '${{ matrix.golang_cross }}'
    runs-on: ubuntu-latest-m
    permissions:
      id-token: write # AWS OIDC JWT
      contents: read # actions/checkout
    strategy:
      fail-fast: false
      matrix:
        golang_cross:
          - 1.21-bookworm
        include:
          - golang_cross: 1.21-bookworm
            goreleaser: 'ci/goreleaser/goreleaser.yml'
            cgo: 0
            rpmvers: 'el/7 el/8 el/9 amazon/2 amazon/2023'
            debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye debian/bookworm'
    outputs:
      tags: ${{ steps.ci_metadata.outputs.tags }}
      commit_author: ${{ steps.set_outputs.outputs.commit_author}}
    steps:
      - name: Checkout of tyk-pump
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: Set some outputs for later
        id: set_outputs
        shell: bash
        env:
          HEAD_REF: ${{github.head_ref}}
        run: |
          echo "commit_author=$(git show -s --format='%ae' HEAD)" >> $GITHUB_OUTPUT
          echo "branch=${HEAD_REF##*/}" >> $GITHUB_OUTPUT
      - uses: docker/setup-qemu-action@v3
      - uses: docker/setup-buildx-action@v3
      - name: Login to DockerHub
        if: startsWith(github.ref, 'refs/tags')
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Login to Cloudsmith
        if: startsWith(github.ref, 'refs/tags')
        uses: docker/login-action@v3
        with:
          registry: docker.tyk.io
          username: ${{ secrets.CLOUDSMITH_USERNAME }}
          password: ${{ secrets.CLOUDSMITH_API_KEY }}
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
      - name: Build
        env:
          NFPM_PASSPHRASE: ${{ secrets.SIGNING_KEY_PASSPHRASE }}
          PKG_SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
        run: |
          echo '#!/bin/sh
          ci/bin/unlock-agent.sh
          git config --global url."https://${{ secrets.ORG_GH_TOKEN }}@github.com".insteadOf "https://github.com"
          git config --global --add safe.directory /go/src/github.com/TykTechnologies/tyk-pump
          goreleaser release --clean -f ${{ matrix.goreleaser }} ${{ !startsWith(github.ref, 'refs/tags/') && ' --snapshot --skip=sign' || '' }}' | tee /tmp/build.sh
          chmod +x /tmp/build.sh
          docker run --rm --privileged -e GITHUB_TOKEN=${{ github.token }} \
          -e GOPRIVATE=github.com/TykTechnologies                                \
          -e DEBVERS='${{ matrix.debvers }}'                               \
          -e RPMVERS='${{ matrix.rpmvers }}'                               \
          -e CGO_ENABLED=${{ matrix.cgo }}                                 \
          -e NFPM_PASSPHRASE="$NFPM_PASSPHRASE"                          \
          -e GPG_FINGERPRINT=12B5D62C28F57592D1575BD51ED14C59E37DAC20            \
          -e PKG_SIGNING_KEY="$PKG_SIGNING_KEY"                                  \
          -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN                              \
          -v ${{github.workspace}}:/go/src/github.com/TykTechnologies/tyk-pump \
          -v /var/run/docker.sock:/var/run/docker.sock                           \
          -v ~/.docker/config.json:/root/.docker/config.json                     \
          -e GOCACHE=/cache/go-build                                             \
          -e GOMODCACHE=/go/pkg/mod                                              \
          -v ~/go/pkg/mod:/go/pkg/mod                                            \
          -v ~/.cache/go-build:/cache/go-build                                   \
          -v /tmp/build.sh:/tmp/build.sh                                         \
          -w /go/src/github.com/TykTechnologies/tyk-pump                      \
          tykio/golang-cross:${{ matrix.golang_cross }} /tmp/build.sh
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
          role-session-name: cipush
          aws-region: eu-central-1
          # Don't mask to pass it across job boundaries
          mask-aws-account-id: false
      - uses: aws-actions/amazon-ecr-login@v2
        id: ecr
        if: ${{ matrix.golang_cross == '1.21-bookworm' }}
        with:
          mask-password: 'true'
      - name: Docker metadata for CI
        id: ci_metadata
        if: ${{ matrix.golang_cross == '1.21-bookworm' }}
        uses: docker/metadata-action@v5
        with:
          images: ${{ steps.ecr.outputs.registry }}/tyk-pump
          flavor: |
            latest=false
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=sha,format=long
            type=semver,pattern={{major}},prefix=v
            type=semver,pattern={{major}}.{{minor}},prefix=v
            type=semver,pattern={{version}},prefix=v
      - name: push image to CI
        if: ${{ matrix.golang_cross == '1.21-bookworm' }}
        uses: docker/build-push-action@v5
        with:
          context: "dist"
          platforms: linux/amd64,linux/arm64
          file: ci/Dockerfile.distroless
          provenance: mode=max
          sbom: true
          push: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          tags: ${{ steps.ci_metadata.outputs.tags }}
          labels: ${{ steps.tag_metadata.outputs.labels }}
      - name: Docker metadata for tag push
        id: tag_metadata
        uses: docker/metadata-action@v5
        with:
          images: |
            tykio/tyk-pump-docker-pub
            docker.tyk.io/tyk-pump/tyk-pump
          flavor: |
            latest=false
            prefix=v
          tags: |
            type=semver,pattern={{major}}
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{version}}
          labels: "org.opencontainers.image.title=tyk-pump  (distroless) \norg.opencontainers.image.description=Tyk Analytics Pump to move analytics data from Redis to any supported back end (multiple back ends can be written to at once).\norg.opencontainers.image.vendor=tyk.io\norg.opencontainers.image.version=${{ github.ref_name }}\n"
      - name: build multiarch image
        if: ${{ matrix.golang_cross == '1.21-bookworm' }}
        uses: docker/build-push-action@v5
        with:
          context: "dist"
          platforms: linux/amd64,linux/arm64
          file: ci/Dockerfile.distroless
          provenance: mode=max
          sbom: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          push: ${{ startsWith(github.ref, 'refs/tags') }}
          tags: ${{ steps.tag_metadata.outputs.tags }}
          labels: ${{ steps.tag_metadata.outputs.labels }}
      - uses: actions/upload-artifact@v4
        if: ${{ matrix.golang_cross == '1.21-bookworm' }}
        with:
          name: deb
          retention-days: 1
          path: |
            dist/*.deb
            !dist/*PAYG*.deb
      - uses: actions/upload-artifact@v4
        if: ${{ matrix.golang_cross == '1.21-bookworm' }}
        with:
          name: rpm
          retention-days: 1
          path: |
            dist/*.rpm
            !dist/*PAYG*.rpm
  test-controller-api:
    needs: goreleaser
    runs-on: ubuntu-latest
    outputs:
      envfiles: ${{ steps.params.outputs.envfiles }}
      pump: ${{ steps.params.outputs.pump }}
      sink: ${{ steps.params.outputs.sink }}
    steps:
      - name: set params
        id: params
        shell: bash
        env:
          # Cover pull_request_target too
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
        run: |
          set -eo pipefail
          endpoint="http://tui.internal.dev.tyk.technology/api/tyk-pump/$BASE_REF/${{ github.event_name}}/api"
          curl="curl -s --retry 5 --retry-delay 10 --fail-with-body --retry-all-errors"
          echo "pump<<EOF
          $($curl ${endpoint}/Pump)
          EOF
          sink<<EOF
          $($curl ${endpoint}/Sink)
          EOF
          envfiles<<EOF
          $($curl ${endpoint}/EnvFiles)
          EOF" | tee -a "$GITHUB_OUTPUT"
  api-tests:
    needs:
      - goreleaser
      - test-controller-api
    runs-on: ubuntu-latest-m-2
    env:
      METADATA_REPORT_PATH: /tmp/metadata.toml
      XUNIT_REPORT_PATH: ${{ github.workspace }}/reports/pytest-report.xml
    permissions:
      id-token: write # This is required for requesting the Github JWT
      contents: read # This is required for actions/checkout
    strategy:
      fail-fast: false
      matrix:
        envfiles: ${{ fromJson(needs.test-controller-api.outputs.envfiles) }}
        sink: ${{ fromJson(needs.test-controller-api.outputs.sink) }}
        include:
          - db: postgres15
            markers: "and not sql"
    steps:
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
          role-session-name: cipush
          aws-region: eu-central-1
      - id: ecr
        uses: aws-actions/amazon-ecr-login@v2
        with:
          mask-password: 'true'
          # Only ${{ github.actor }} has access
          # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys
      - name: Setup tmate session only in debug mode
        uses: mxschmitt/action-tmate@v3
        if: runner.debug == '1'
        with:
          detached: true
          limit-access-to-actor: true
      - name: fetch env from tyk-ci
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          gh release download --repo github.com/TykTechnologies/tyk-ci -p 'ci-env.tgz' -O env.tgz
          tar xzvf env.tgz
      - name: env up
        shell: bash
        working-directory: auto
        id: env_up
        env:
          pull_policy: 'if_not_present'
          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
        run: |
          match_tag=${{steps.ecr.outputs.registry}}/$REPO:$BASE_REF
          tags=(${{ needs.goreleaser.outputs.tags }})
          docker run -q --rm -v ~/.docker/config.json:/root/.docker/config.json tykio/gromit policy match ${tags[0]} ${match_tag} 2>versions.env
          echo '# alfa and beta have to come after the override
          tyk_alfa_image=$tyk_image
          tyk_beta_image=$tyk_image
          ECR=${{steps.ecr.outputs.registry}}
          tyk_sink_image=${{matrix.sink}}
          confs_dir=./pro-ha
          env_file=local-${{ matrix.envfiles.db }}.env' >> versions.env
          echo "::group::versions"
          cat versions.env
          echo "::endgroup::"
          # Add Tyk component config variations to $env_file
          cat confs/${{ matrix.envfiles.config }}.env >> local-${{ matrix.envfiles.db }}.env
          # bring up env, the project name is important
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml  --env-file versions.env --profile master-datacenter up --quiet-pull -d
          ./dash-bootstrap.sh http://localhost:3000
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml  --env-file versions.env --profile slave-datacenter up --quiet-pull -d
      - name: Run tests
        working-directory: auto
        id: test_execution
        env:
          # Cover pull_request_target too
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name }}
        run: |
          # Generate report id
          echo "id=$(date +%s%N)" >> $GITHUB_OUTPUT
          # Run tests
          set -o pipefail
          echo "### API tests ${{ matrix.envfiles.db }} ${{ matrix.envfiles.conf }}" >> $GITHUB_STEP_SUMMARY
          if docker run --rm --network auto_default --env-file pytest.env -v ${{ github.workspace }}/reports:/app/reports \
            ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:$BASE_REF \
            pytest -c pytest_ci.ini --junitxml=./${XUNIT_REPORT_PATH#"${{ github.workspace }}"} --ci -m "not local and not dind ${{ matrix.markers }}" | tee tests.out; then
              echo "All tests passed!" >> $GITHUB_STEP_SUMMARY
          else
            echo "::error title=API tests ${{ matrix.envfiles.db }} ${{ matrix.envfiles.conf }}::Test execution failed"
            cat tests.out >> $GITHUB_STEP_SUMMARY
            exit 1
          fi
      - name: Generate metadata
        if: always() && steps.test_execution.outcome != 'skipped'
        id: metadata_report
        env:
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref}}
          REPORT_NAME: ${{ github.repository }}_${{ github.run_id }}_${{ github.run_attempt }}-${{steps.test_execution.outputs.id}}
        run: |
          # Generate metadata report
          echo "[metadata]
          repo = ${{ github.repository }}
          branch = ${{ github.ref }}
          commit = ${{ github.sha }}
          test_suite_version = $BASE_REF
          test_suite_name = ${{ github.job }}
          test_suite_run = ${{ github.run_id }}-${{ github.run_attempt }}
          db = ${{ matrix.envfiles.db }}
          conf = ${{ matrix.envfiles.config }}
          cache = ${{ matrix.envfiles.cache }}
          pump_compatibility = ${{ matrix.pump }}
          sink_compatibility = ${{ matrix.sink }}
          " > ${METADATA_REPORT_PATH}
          # Print metadata report
          if [[ "${{ runner.debug }}" == "1" ]]; then
            echo "::group::metadata report"
            cat ${METADATA_REPORT_PATH}
            echo "::endgroup::"
          fi
          #Upload xunit report
          aws s3 cp ${XUNIT_REPORT_PATH}  s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.xml
          #Upload metadata report
          aws s3 cp ${METADATA_REPORT_PATH} s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.metadata.toml
      - name: Docker logs for all components
        if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
        working-directory: auto
        env:
          pull_policy: 'if_not_present'
          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
          ECR: ${{ steps.ecr.outputs.registry }}
        run: |
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml  --env-file versions.env --profile all logs | sort > ${{ github.workspace }}/docker-compose.log
          echo "::group::DockerLogs"
          cat ${{ github.workspace }}/docker-compose.log
          echo "::endgroup::"
      - name: Upload Artifact
        uses: actions/upload-artifact@v4
        if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
        with:
          name: docker-compose-logs-${{ github.job }}-${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}-${{ github.run_id }}
          path: ${{ github.workspace }}/docker-compose.log
          retention-days: 3
          overwrite: true
      - name: Archive Integration tests report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: api-test-report-${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}-${{ github.run_id }}
          retention-days: 3
          path: ${{ github.workspace }}/reports
          overwrite: true
  upgrade-deb:
    services:
      httpbin.org:
        image: kennethreitz/httpbin
    runs-on: ubuntu-latest
    needs: goreleaser
    strategy:
      fail-fast: false
      matrix:
        arch:
          - amd64
          - arm64
        distro:
          - ubuntu:xenial
          - ubuntu:bionic
          - ubuntu:focal
          - ubuntu:jammy
          - debian:bullseye
          - debian:bookworm
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - uses: actions/download-artifact@v4
        with:
          name: deb
      - uses: docker/setup-qemu-action@v3
      - uses: docker/setup-buildx-action@v3
      - name: generate dockerfile
        run: |
          echo 'FROM ${{ matrix.distro }}
          ARG TARGETARCH
          COPY tyk-pump*_${TARGETARCH}.deb /tyk-pump.deb
          RUN apt-get update && apt-get install -y curl
          RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.deb.sh | bash && apt-get install -y tyk-pump=1.6.0
          RUN dpkg -i tyk-pump.deb
          ' > Dockerfile
      - name: install on ${{ matrix.distro }}
        uses: docker/build-push-action@v5
        with:
          context: "."
          platforms: linux/${{ matrix.arch }}
          file: Dockerfile
          push: false
  upgrade-rpm:
    services:
      httpbin.org:
        image: kennethreitz/httpbin
    needs: goreleaser
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        distro:
          - amazonlinux:2023
          - registry.access.redhat.com/ubi8/ubi
          - registry.access.redhat.com/ubi9/ubi
          - amazonlinux:2
          - registry.access.redhat.com/ubi7/ubi
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - uses: actions/download-artifact@v4
        with:
          name: rpm
      - uses: docker/setup-buildx-action@v3
      - name: generate dockerfile
        run: |
          echo 'FROM ${{ matrix.distro }}
          COPY tyk-pump*.x86_64.rpm /tyk-pump.rpm
          RUN command -v curl || yum install -y curl
          RUN command -v useradd || yum install -y shadow-utils
          RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.rpm.sh | bash && yum install -y tyk-pump-1.6.0-1
          RUN curl https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 -o tyk-pump.key && rpm --import tyk-pump.key
          RUN rpm --checksig tyk-pump.rpm
          RUN rpm -Uvh --force tyk-pump.rpm
          ' > Dockerfile
      - name: install on ${{ matrix.distro }}
        uses: docker/build-push-action@v5
        with:
          context: "."
          file: Dockerfile
          push: false
  sbom:
    needs: goreleaser
    uses: TykTechnologies/github-actions/.github/workflows/sbom.yaml@main
    secrets:
      DEPDASH_URL: ${{ secrets.DEPDASH_URL }}
      DEPDASH_KEY: ${{ secrets.DEPDASH_KEY }}
      ORG_GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}