release.yml

# Generated by: gromit policy

# Distribution channels covered by this workflow
# - Ubuntu and Debian
# - RHEL and AL
# - docker hub
# - devenv ECR
# - Cloudsmith

name: Release
on:
  # Trigger release every monday at midnight for master CI images
  schedule:
    - cron: "0 0 * * 1"
  pull_request:
  push:
    branches:
      - master
      - release-**
    tags:
      - 'v*'
env:
  GOPRIVATE: github.com/TykTechnologies
  VARIATION: prod
  DOCKER_BUILD_SUMMARY: false
  DOCKER_BUILD_RECORD_UPLOAD: false
  # startsWith covers pull_request_target too
  BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
jobs:
  goreleaser:
    name: '${{ matrix.golang_cross }}'
    runs-on: ubuntu-latest-m
    permissions:
      id-token: write # AWS OIDC JWT
      contents: read # actions/checkout
    strategy:
      fail-fast: false
      matrix:
        golang_cross:
          - 1.22-bullseye
        include:
          - golang_cross: 1.22-bullseye
            goreleaser: 'ci/goreleaser/goreleaser.yml'
            cgo: 1
            rpmvers: 'el/7 el/8 el/9 amazon/2 amazon/2023'
            debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye debian/bookworm'
    outputs:
      tags: ${{ steps.ci_metadata.outputs.tags }}
      commit_author: ${{ steps.set_outputs.outputs.commit_author}}
    steps:
      - name: Checkout of tyk
        uses: actions/checkout@v4
        with:
          fetch-depth: 1
      - name: Set some outputs for later
        id: set_outputs
        shell: bash
        env:
          HEAD_REF: ${{github.head_ref}}
        run: |
          echo "commit_author=$(git show -s --format='%ae' HEAD)" >> $GITHUB_OUTPUT
          echo "branch=${HEAD_REF##*/}" >> $GITHUB_OUTPUT
      - uses: docker/setup-qemu-action@v3
      - uses: docker/setup-buildx-action@v3
      - name: Login to DockerHub
        if: startsWith(github.ref, 'refs/tags')
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Login to Cloudsmith
        if: startsWith(github.ref, 'refs/tags')
        uses: docker/login-action@v3
        with:
          registry: docker.tyk.io
          username: ${{ secrets.CLOUDSMITH_USERNAME }}
          password: ${{ secrets.CLOUDSMITH_API_KEY }}
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/go-build
            ~/go/pkg/mod
          key: ${{ runner.os }}-${{ matrix.golang_cross }}-go-${{ hashFiles('**/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.golang_cross }}-go-${{ hashFiles('**/go.sum') }}
      - name: Build
        env:
          NFPM_PASSPHRASE: ${{ secrets.SIGNING_KEY_PASSPHRASE }}
          PKG_SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
        run: |
          echo '#!/bin/sh
          ci/bin/unlock-agent.sh
          git config --global url."https://${{ secrets.ORG_GH_TOKEN }}@github.com".insteadOf "https://github.com"
          git config --global --add safe.directory /go/src/github.com/TykTechnologies/tyk
          goreleaser release --clean -f ${{ matrix.goreleaser }} ${{ !startsWith(github.ref, 'refs/tags/') && ' --snapshot --skip=sign' || '' }}' | tee /tmp/build.sh
          chmod +x /tmp/build.sh
          docker run --rm --privileged -e GITHUB_TOKEN=${{ github.token }} \
          -e GOPRIVATE=github.com/TykTechnologies                                \
          -e DEBVERS='${{ matrix.debvers }}'                               \
          -e RPMVERS='${{ matrix.rpmvers }}'                               \
          -e CGO_ENABLED=${{ matrix.cgo }}                                 \
          -e NFPM_PASSPHRASE="$NFPM_PASSPHRASE"                          \
          -e GPG_FINGERPRINT=12B5D62C28F57592D1575BD51ED14C59E37DAC20            \
          -e PKG_SIGNING_KEY="$PKG_SIGNING_KEY"                                  \
          -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN                              \
          -v ${{github.workspace}}:/go/src/github.com/TykTechnologies/tyk \
          -v /var/run/docker.sock:/var/run/docker.sock                           \
          -v ~/.docker/config.json:/root/.docker/config.json                     \
          -e GOCACHE=/cache/go-build                                             \
          -e GOMODCACHE=/go/pkg/mod                                              \
          -v ~/go/pkg/mod:/go/pkg/mod                                            \
          -v ~/.cache/go-build:/cache/go-build                                   \
          -v /tmp/build.sh:/tmp/build.sh                                         \
          -w /go/src/github.com/TykTechnologies/tyk                      \
          tykio/golang-cross:${{ matrix.golang_cross }} /tmp/build.sh
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
          role-session-name: cipush
          aws-region: eu-central-1
          # Don't mask to pass it across job boundaries
          mask-aws-account-id: false
      - uses: aws-actions/amazon-ecr-login@v2
        id: ecr
        if: ${{ matrix.golang_cross == '1.22-bullseye' }}
        with:
          mask-password: 'true'
      - name: Docker metadata for CI
        id: ci_metadata
        if: ${{ matrix.golang_cross == '1.22-bullseye' }}
        uses: docker/metadata-action@v5
        with:
          images: ${{ steps.ecr.outputs.registry }}/tyk
          flavor: |
            latest=false
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=sha,format=long
            type=semver,pattern={{major}},prefix=v
            type=semver,pattern={{major}}.{{minor}},prefix=v
            type=semver,pattern={{version}},prefix=v
      - name: push image to CI
        if: ${{ matrix.golang_cross == '1.22-bullseye' }}
        uses: docker/build-push-action@v6
        with:
          context: "dist"
          platforms: linux/amd64,linux/arm64
          file: ci/Dockerfile.distroless
          provenance: mode=max
          sbom: true
          push: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          tags: ${{ steps.ci_metadata.outputs.tags }}
          labels: ${{ steps.tag_metadata.outputs.labels }}
      - name: Docker metadata for tag push
        id: tag_metadata
        uses: docker/metadata-action@v5
        with:
          images: |
            tykio/tyk-gateway
            docker.tyk.io/tyk-gateway/tyk-gateway
          flavor: |
            latest=false
            prefix=v
          tags: |
            type=semver,pattern={{major}}
            type=semver,pattern={{major}}.{{minor}}
            type=semver,pattern={{version}}
          labels: "org.opencontainers.image.title=tyk-gateway  (distroless) \norg.opencontainers.image.description=Tyk Open Source API Gateway written in Go, supporting REST, GraphQL, TCP and gRPC protocols\norg.opencontainers.image.vendor=tyk.io\norg.opencontainers.image.version=${{ github.ref_name }}\n"
      - name: build multiarch image
        if: ${{ matrix.golang_cross == '1.21-bullseye' }}
        uses: docker/build-push-action@v5
        with:
          context: "dist"
          platforms: linux/amd64,linux/arm64
          file: ci/Dockerfile.distroless
          provenance: mode=max
          sbom: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          push: ${{ startsWith(github.ref, 'refs/tags') }}
          tags: ${{ steps.tag_metadata.outputs.tags }}
          labels: ${{ steps.tag_metadata.outputs.labels }}
      - uses: actions/upload-artifact@v4
        with:
          name: deb
          retention-days: 1
          path: |
            dist/*.deb
            !dist/*PAYG*.deb
      - uses: actions/upload-artifact@v4
        with:
          name: rpm
          retention-days: 1
          path: |
            dist/*.rpm
            !dist/*PAYG*.rpm
            !dist/*fips*.rpm
  test-controller-api:
    needs:
      - goreleaser
    runs-on: ubuntu-latest
    outputs:
      envfiles: ${{ steps.params.outputs.envfiles }}
      pump: ${{ steps.params.outputs.pump }}
      sink: ${{ steps.params.outputs.sink }}
    steps:
      - name: set params
        id: params
        shell: bash
        run: |
          set -eo pipefail
          curl -s --retry 5 --retry-delay 10 --fail-with-body "http://tui.internal.dev.tyk.technology/v2/$VARIATION/tyk/$BASE_REF/${{ github.event_name}}/api.gho" | tee -a "$GITHUB_OUTPUT"
          if ! [[ $VARIATION =~ prod ]] ;then
              echo "::warning file=.github/workflows/release.yml,line=24,col=1,endColumn=8::Using non-prod variation"
              echo "### :warning: You are using VARIATION=${VARIATION} in test-controller-api" >> $GITHUB_STEP_SUMMARY
          fi
  api-tests:
    needs:
      - test-controller-api
      - goreleaser
    runs-on: ubuntu-latest-m-2
    env:
      XUNIT_REPORT_PATH: ${{ github.workspace}}/test-results.xml
    permissions:
      id-token: write # This is required for requesting the Github JWT
      contents: read # This is required for actions/checkout
    strategy:
      fail-fast: false
      matrix:
        envfiles: ${{ fromJson(needs.test-controller-api.outputs.envfiles) }}
        pump: ${{ fromJson(needs.test-controller-api.outputs.pump) }}
        sink: ${{ fromJson(needs.test-controller-api.outputs.sink) }}
        exclude:
          - pump: tykio/tyk-pump-docker-pub:v1.8
            sink: $ECR/tyk-sink:master
          - pump: $ECR/tyk-pump:master
            sink: tykio/tyk-mdcb-docker:v2.4
    steps:
      - uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk
          role-session-name: cipush
          aws-region: eu-central-1
      - id: ecr
        uses: aws-actions/amazon-ecr-login@v2
        with:
          mask-password: 'true'
      - name: Setup tmate session only in debug mode
        uses: mxschmitt/action-tmate@v3
        if: runner.debug == '1'
        with:
          detached: true
          limit-access-to-actor: true
          # Only ${{ github.actor }} has access
          # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys
      - name: fetch env from tyk-pro
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          gh release download --repo github.com/tyklabs/tyk-pro --archive tar.gz -O env.tgz
          mkdir auto && tar --strip-components=1 -C auto -xzvf env.tgz
      - name: env up
        shell: bash
        working-directory: auto
        id: env_up
        env:
          pull_policy: 'if_not_present'
          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
        run: |
          match_tag=${{steps.ecr.outputs.registry}}/tyk:$BASE_REF
          tags=(${{ needs.goreleaser.outputs.tags }})
          set -eaxo pipefail
          docker run -q --rm -v ~/.docker/config.json:/root/.docker/config.json tykio/gromit policy match ${tags[0]} ${match_tag} 2>versions.env
          echo '# alfa and beta have to come after the override
          tyk_alfa_image=$tyk_image
          tyk_beta_image=$tyk_image
          ECR=${{steps.ecr.outputs.registry}}
          tyk_pump_image=${{matrix.pump}}

          tyk_sink_image=${{matrix.sink}}
          confs_dir=./pro-ha
          env_file=local.env' >> versions.env
          cat ./confs/${{ matrix.envfiles.config }}.env local-${{ matrix.envfiles.db }}.env > local.env
          echo "::group::versions"
          cat versions.env local.env
          echo "::endgroup::"
          # bring up env, the project name is important
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d
          ./dash-bootstrap.sh http://localhost:3000
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d
          echo "$(cat pytest.env | grep USER_API_SECRET)" >> $GITHUB_OUTPUT
      - uses: actions/checkout@v4
        with:
          repository: TykTechnologies/tyk-analytics
          path: tyk-analytics
          token: ${{ secrets.ORG_GH_TOKEN }}
          fetch-depth: 1
          sparse-checkout: tests/api
      - name: Branch for test code
        id: timestamp
        working-directory: tyk-analytics/tests/api
        run: |
          # Generate report id
          git fetch --no-tags --depth 1 origin "refs/heads/${BASE_REF}:refs/remotes/origin/${BASE_REF}"
          git switch $BASE_REF
          echo "ts=$(date +%s%N)" >> $GITHUB_OUTPUT
      - uses: actions/setup-python@v5
        with:
          cache: 'pip'
          python-version: '3.10'
      - name: Run API tests
        id: test_execution
        working-directory: tyk-analytics/tests/api
        run: |
          pytest="pytest --ci --random-order --force-flaky --no-success-flaky-report --maxfail=3 --junitxml=${XUNIT_REPORT_PATH} --cache-clear --ignore=./tests/mdcb -v --log-cli-level=ERROR"
          pip install -r requirements.txt
          cat >pytest.env <<-EOF
          TYK_TEST_BASE_URL=http://localhost:3000/
          TYK_TEST_GW_URL=https://localhost:8080/
          TYK_TEST_GW_1_ALFA_URL=https://localhost:8181/
          TYK_TEST_GW_1_BETA_URL=https://localhost:8182/
          TYK_TEST_GW_2_ALFA_URL=https://localhost:8281/
          TYK_TEST_GW_2_BETA_URL=https://localhost:8282/
          TYK_TEST_MONGODB=localhost:27017
          TYK_TEST_REDIS=localhost
          TYK_TEST_DB_ADMIN=12345
          TYK_TEST_GW_SECRET=352d20ee67be67f6340b4c0605b044b7
          TYK_TEST_DB_NAME=tyk_analytics
          TYK_TEST_FEDERATION_HOST=federation
          TYK_TEST_GRAPHQL_FAKER_HOST=graphql-faker
          GATEWAY_CONTAINER_NAME=tyk
          USER_API_SECRET=${{ steps.env_up.outputs.USER_API_SECRET }}
          EOF
          env $(cat pytest.env | xargs) $pytest -m "${{ matrix.envfiles.apimarkers }}"
      - name: Generate metadata and upload test reports
        id: metadata_report
        if: always() && (steps.test_execution.conclusion != 'skipped')
        env:
          REPORT_NAME: ${{ github.repository }}_${{ github.run_id }}_${{ github.run_attempt }}-${{steps.timestamp.outputs.ts}}
          METADATA_REPORT_PATH: metadata.toml
        run: |
          # Generate metadata report
          set -eo pipefail
          echo "[metadata]
          repo = ${{ github.repository }}
          branch = ${{ github.ref }}
          commit = ${{ github.sha }}
          test_suite_version = $BASE_REF
          test_suite_name = ${{ github.job }}
          test_suite_run = ${{ github.run_id }}-${{ github.run_attempt }}
          db = ${{ matrix.envfiles.db }}
          conf = ${{ matrix.envfiles.config }}
          cache = ${{ matrix.envfiles.cache }}
          pump_compatibility = ${{ matrix.pump }}
          sink_compatibility = ${{ matrix.sink }}
          " | tee ${METADATA_REPORT_PATH}
          aws s3 cp ${XUNIT_REPORT_PATH}  s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.xml
          aws s3 cp ${METADATA_REPORT_PATH} s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.metadata.toml
      - name: Docker logs for all components
        if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
        working-directory: auto
        env:
          pull_policy: 'if_not_present'
          GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}
          TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
          TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
          ECR: ${{ steps.ecr.outputs.registry }}
        run: |
          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.envfiles.db }}.yml -f ${{ matrix.envfiles.cache }}.yml --env-file versions.env --profile all logs | sort > ${{ github.workspace }}/docker-compose.log
          echo "::group::DockerLogs"
          cat ${{ github.workspace }}/docker-compose.log
          echo "::endgroup::"
      - name: Upload compose logs
        uses: actions/upload-artifact@v4
        if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
        with:
          name: docker-compose-logs-${{ github.job }}-${{ matrix.envfiles.db }}-${{ matrix.envfiles.conf }}-${{ github.run_id }}
          path: ${{ github.workspace }}/docker-compose.log
          retention-days: 3
          overwrite: true
  test-controller-distros:
    needs:
      - goreleaser
    runs-on: ubuntu-latest
    outputs:
      deb: ${{ steps.params.outputs.deb }}
      rpm: ${{ steps.params.outputs.rpm }}
    steps:
      - name: set params
        id: params
        shell: bash
        env:
          # startsWith covers pull_request_target too
          BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref_name}}
        run: |
          set -eo pipefail
          curl -s --retry 5 --retry-delay 10 --fail-with-body "http://tui.internal.dev.tyk.technology/v2/$VARIATION/tyk/$BASE_REF/${{ github.event_name}}/api/Distros.gho" | tee -a "$GITHUB_OUTPUT"
          if ! [[ $VARIATION =~ prod ]];then
            echo "::warning file=.github/workflows/release.yml,line=24,col=1,endColumn=8::Using test variation"
          fi
  upgrade-deb:
    services:
      httpbin.org:
        image: kennethreitz/httpbin
    runs-on: ubuntu-latest
    needs:
      - test-controller-distros
    strategy:
      fail-fast: true
      matrix:
        arch:
          - amd64
          - arm64
        distro: ${{ fromJson(needs.test-controller-distros.outputs.deb) }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
          sparse-checkout: ci
      - uses: actions/download-artifact@v4
        with:
          name: deb
      - uses: docker/setup-qemu-action@v3
      - uses: docker/setup-buildx-action@v3
      - name: generate dockerfile
        run: |
          echo 'FROM ${{ matrix.distro }}
          ARG TARGETARCH
          COPY tyk-gateway*_${TARGETARCH}.deb /tyk-gateway.deb
          RUN apt-get update && apt-get install -y curl
          RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-gateway/script.deb.sh | bash && apt-get install -y tyk-gateway=3.0.8
          RUN dpkg -i /tyk-gateway.deb

          RUN /opt/tyk-gateway/install/setup.sh --listenport=8080 --redishost=localhost --redisport=6379 --domain=""
          COPY ci/tests/api-functionality/data/api.json /opt/tyk-gateway/apps/
          CMD ["/opt/tyk-gateway/tyk", "--conf", "/opt/tyk-gateway/tyk.conf"]
          ' | tee Dockerfile
      - name: install on ${{ matrix.distro }}
        uses: docker/build-push-action@v6
        with:
          context: "."
          platforms: linux/${{ matrix.arch }}
          build-args: |
            RHELARCH=${{ startsWith(matrix.arch, 'arm64') && 'aarch64' || 'x86_64' }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          file: Dockerfile
          push: false
          tags: test-${{ matrix.distro }}-${{ matrix.arch }}
          load: true
      - name: Test the built container image with api functionality test.
        run: |
          docker run -d -p8080:8080 --network ${{ job.container.network }} --rm test-${{ matrix.distro }}-${{ matrix.arch }}
          sleep 2
          ./ci/tests/api-functionality/api_test.sh
  upgrade-rpm:
    services:
      httpbin.org:
        image: kennethreitz/httpbin
    runs-on: ubuntu-latest
    needs:
      - test-controller-distros
    strategy:
      fail-fast: true
      matrix:
        arch:
          - amd64
          - arm64
        distro: ${{ fromJson(needs.test-controller-distros.outputs.rpm) }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 1
          sparse-checkout: ci
      - uses: actions/download-artifact@v4
        with:
          name: rpm
      - uses: docker/setup-qemu-action@v3
      - uses: docker/setup-buildx-action@v3
      - name: generate dockerfile
        run: |
          echo 'FROM ${{ matrix.distro }}
          ARG RHELARCH
          COPY tyk-gateway*.${RHELARCH}.rpm /tyk-gateway.rpm
          RUN command -v curl || yum install -y curl
          RUN command -v useradd || yum install -y shadow-utils
          RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-gateway/script.rpm.sh | bash && yum install -y tyk-gateway-3.0.8-1
          RUN curl https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 -o tyk-gateway.key && rpm --import tyk-gateway.key
          RUN rpm --checksig /tyk-gateway.rpm
          RUN rpm -Uvh --force /tyk-gateway.rpm

          RUN /opt/tyk-gateway/install/setup.sh --listenport=8080 --redishost=localhost --redisport=6379 --domain=""
          COPY ci/tests/api-functionality/data/api.json /opt/tyk-gateway/apps/
          CMD ["/opt/tyk-gateway/tyk", "--conf", "/opt/tyk-gateway/tyk.conf"]
          ' | tee Dockerfile
      - name: install on ${{ matrix.distro }}
        uses: docker/build-push-action@v6
        with:
          context: "."
          platforms: linux/${{ matrix.arch }}
          build-args: |
            RHELARCH=${{ startsWith(matrix.arch, 'arm64') && 'aarch64' || 'x86_64' }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          file: Dockerfile
          push: false
          tags: test-${{ matrix.distro }}-${{ matrix.arch }}
          load: true
      - name: Test the built container image with api functionality test.
        run: "docker run -d -p8080:8080 --network ${{ job.container.network }} --rm test-${{ matrix.distro }}-${{ matrix.arch }}\nsleep 2\n./ci/tests/api-functionality/api_test.sh \n"
  release-tests:
    needs:
      - goreleaser
    permissions:
      id-token: write # This is required for requesting the JWT
      contents: read # This is required for actions/checkout
    uses: ./.github/workflows/release-tests.yml
    secrets: inherit
  sbom:
    needs: goreleaser
    uses: TykTechnologies/github-actions/.github/workflows/sbom.yaml@main
    secrets:
      DEPDASH_URL: ${{ secrets.DEPDASH_URL }}
      DEPDASH_KEY: ${{ secrets.DEPDASH_KEY }}
      ORG_GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }}