From ea9e83d5b453ef731f65e782a30ef71a4d08d104 Mon Sep 17 00:00:00 2001 From: Gromit Date: Thu, 26 Oct 2023 05:43:34 +0000 Subject: [PATCH] Auto generated from templates by gromit --- .github/dependabot.yml | 2 +- .github/workflows/release.yml | 238 +++++++++++++++++++++++----------- ci/Dockerfile.std | 8 +- ci/aws/byol.pkr.hcl | 2 +- ci/goreleaser/goreleaser.yml | 16 ++- ci/install/before_install.sh | 2 +- ci/install/post_install.sh | 2 +- ci/install/post_remove.sh | 2 +- ci/install/post_trans.sh | 2 +- 9 files changed, 184 insertions(+), 90 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index bffb4d534..69a0a3147 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,6 +1,6 @@ # Generated by: gromit policy -# Generated on: Wed Jul 26 08:25:03 UTC 2023 +# Generated on: Thu Oct 26 05:43:34 UTC 2023 version: 2 updates: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f6509fa51..26d6ccafb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,12 +1,11 @@ # Generated by: gromit policy -# Generated on: Wed Jul 26 08:25:03 UTC 2023 +# Generated on: Thu Oct 26 05:43:34 UTC 2023 # Distribution channels covered by this workflow # - Ubuntu and Debian # - RHEL and AL # - docker hub # - devenv ECR -# - AWS mktplace (not active atm) # - Cloudsmith name: Release @@ -29,6 +28,9 @@ jobs: name: '${{ matrix.golang_cross }}' runs-on: ubuntu-latest container: 'tykio/golang-cross:${{ matrix.golang_cross }}' + permissions: + id-token: write # AWS OIDC JWT + contents: read # actions/checkout strategy: fail-fast: false matrix: @@ -37,9 +39,9 @@ jobs: - golang_cross: 1.19-bullseye goreleaser: 'ci/goreleaser/goreleaser.yml' rpmvers: 'el/7 el/8 el/9 amazon/2 amazon/2023' - debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye' + debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye debian/bookworm' outputs: - tag: ${{ steps.targets.outputs.tag }} + tags: ${{ steps.metadata.outputs.tags }} steps: - name: Fix private module deps @@ -74,32 +76,24 @@ jobs: username: ${{ secrets.CLOUDSMITH_USERNAME }} password: ${{ secrets.CLOUDSMITH_API_KEY }} - - name: Unlock agent and set tag - id: targets - shell: bash + - name: Unlock agent env: NFPM_STD_PASSPHRASE: ${{ secrets.SIGNING_KEY_PASSPHRASE }} GPG_FINGERPRINT: 12B5D62C28F57592D1575BD51ED14C59E37DAC20 PKG_SIGNING_KEY: ${{ secrets.SIGNING_KEY }} - run: | + run: ci/bin/unlock-agent.sh - current_tag=${GITHUB_REF##*/} - echo "tag=${current_tag}" >> $GITHUB_OUTPUT - - name: Delete old release assets - if: startsWith(github.ref, 'refs/tags') - uses: mknejp/delete-release-assets@v1 + + + - uses: actions/cache@v3 with: - token: ${{ github.token }} - tag: ${{ github.ref }} - fail-if-no-assets: false - fail-if-no-release: false - assets: | - *.deb - *.rpm - *.tar.gz - *.txt.sig - *.txt + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- - uses: goreleaser/goreleaser-action@v4 with: @@ -116,6 +110,48 @@ jobs: DEBVERS: ${{ matrix.debvers }} RPMVERS: ${{ matrix.rpmvers }} PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} + - uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk + role-session-name: cipush + aws-region: eu-central-1 + # Don't mask to pass it across job boundaries + mask-aws-account-id: false + + - uses: aws-actions/amazon-ecr-login@v1 + id: ecr + with: + mask-password: 'true' + + - name: Docker metadata for CI + id: metadata + uses: docker/metadata-action@v4 + with: + images: ${{ steps.ecr.outputs.registry }}/tyk-pump + flavor: | + latest=false + tags: | + type=ref,event=branch + type=ref,event=pr + type=sha,format=long + type=semver,pattern=v{{major}}.{{minor}},prefix=v + type=semver,pattern=v{{version}},prefix=v + + - name: CI push + shell: bash + env: + t: ${{ steps.metadata.outputs.tags }} + build_tag: ${{ startswith(github.ref, 'refs/tags') && github.ref_name || 'v0.0.0' }} + run: | + set +e + IFS=$'\n' tags=($t) + for tag in "${tags[@]}"; do + for arch in amd64 arm64; do + docker tag tykio/tyk-pump-docker-pub:${build_tag}-${arch} ${tag}-${arch} && docker push ${tag}-${arch} + done + docker manifest create ${tag} ${tag}-amd64 ${tag}-arm64 && docker manifest push ${tag} + done + - uses: actions/upload-artifact@v3 with: name: deb @@ -131,73 +167,102 @@ jobs: path: | dist/*.rpm !dist/*PAYG*.rpm - ci: - needs: - - goreleaser + api-tests: + needs: goreleaser runs-on: ubuntu-latest permissions: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout + strategy: + fail-fast: false + matrix: + conf: [ "sha256", "murmur64" ] + db: [ "mongo44", "postgres15" ] + include: + - db: postgres15 + markers: "and not sql" steps: - - name: Shallow checkout of tyk-pump - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 1 + - name: Checkout of tyk-automated-tests + uses: actions/checkout@v3 + with: + repository: TykTechnologies/tyk-automated-tests + token: ${{ secrets.ORG_GH_TOKEN }} + ref: ${{ startsWith(github.ref_name, 'release-') && github.ref_name || 'master' }} + path: tyk-automated-tests + - uses: aws-actions/configure-aws-credentials@v2 with: role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk role-session-name: cipush aws-region: eu-central-1 - - name: Login to Amazon ECR - id: login-ecr + - id: ecr uses: aws-actions/amazon-ecr-login@v1 - - - uses: actions/download-artifact@v3 with: - name: deb + mask-password: 'true' - - name: Docker metadata - id: metadata - uses: docker/metadata-action@v4 + # Only ${{ github.actor }} has access + # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys + - name: Setup tmate session only in debug mode + uses: mxschmitt/action-tmate@v3 + if: ${{ runner.debug == '1' }} with: - images: ${{ steps.login-ecr.outputs.registry }}/tyk-pump - flavor: | - latest=false - prefix=v - tags: | - type=semver,pattern=v{{version}} - type=semver,pattern=v{{major}}.{{minor}} - type=semver,pattern=v{{major}} - type=ref,event=branch - - - uses: docker/setup-qemu-action@v2 + detached: true + limit-access-to-actor: true - - uses: docker/setup-buildx-action@v2 - - - name: CI build - uses: docker/build-push-action@v4 - with: - push: true - context: "." - file: ci/Dockerfile.std - platforms: linux/amd64,linux/arm64 - tags: | - ${{ steps.metadata.outputs.tags }} - ${{ steps.login-ecr.outputs.registry }}/tyk-pump:${{ needs.goreleaser.outputs.tag }} - ${{ steps.login-ecr.outputs.registry }}/tyk-pump:${{ github.sha }} + - name: env up + shell: bash + working-directory: tyk-automated-tests/ci/auto + id: env_up + env: + t: ${{ needs.goreleaser.outputs.tags }} + pull_policy: 'if_not_present' + # gw and dash use the same branch names + gw_dash_image_tag: ${{ ( (github.repository == 'TykTechnologies/tyk' || github.repository == 'TykTechnologies/tyk-analytics') && startsWith(github.ref_name, 'release-') ) && github.ref_name || 'master' }} + pump_image_tag: ${{ ( github.repository == 'TykTechnologies/tyk-pump' && startsWith(github.ref_name, 'release-') ) && github.ref_name || 'master' }} + sink_image_tag: ${{ ( github.repository == 'TykTechnologies/tyk-sink' && startsWith(github.ref_name, 'release-') ) && github.ref_name || 'master' }} + GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }} + TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }} + TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }} + run: | + echo CI tags: $t + tags=($t) + echo First tag: ${tags[0]} + # Start customising the env + echo "registry=${{ steps.ecr.outputs.registry }} + tyk_image=\${registry}/tyk:${gw_dash_image_tag} + tyk_analytics_image=\${registry}/tyk-analytics:${gw_dash_image_tag} + tyk_sink_image=\${registry}/tyk-sink:${sink_image_tag} + tyk_pump_image=\${registry}/tyk-pump:${pump_image_tag} + # override default above with just built tag + tyk_image=${tags[0]} + tyk_alfa_image=${tags[0]} + tyk_beta_image=${tags[0]} + # base dir for config files + confs_dir=./pro-ha + # pick database to use + env_file=local-${{ matrix.db }}.env + " > versions.env + # Add Tyk component config variations to $env_file + cat confs/${{ matrix.conf }}.env >> local-${{ matrix.db }}.env + # bring up env, the project name is important + docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d + ./dash-bootstrap.sh http://localhost:3000 + docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d - sbom: - needs: ci - uses: TykTechnologies/github-actions/.github/workflows/sbom.yaml@main - secrets: - TF_API_TOKEN: ${{ secrets.TF_API_TOKEN }} - DEPDASH_URL: ${{ secrets.DEPDASH_URL }} - DEPDASH_KEY: ${{ secrets.DEPDASH_KEY }} - ORG_GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }} + - name: Run tests + working-directory: tyk-automated-tests/ci/auto + run: | + docker run --rm --network auto_default --env-file pytest.env \ + ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:${{ startsWith(github.ref_name, 'release-') && github.ref_name || 'master' }} \ + pytest -c pytest_ci.ini --ci -m "not local and not dind ${{ matrix.markers }}" +# TODO: PR comment, using just one comment and updating, not a new comment each time upgrade-deb: if: startsWith(github.ref, 'refs/tags') @@ -213,10 +278,11 @@ jobs: - amd64 - arm64 distro: - - ubuntu:jammy - ubuntu:bionic - ubuntu:focal + - ubuntu:jammy - debian:bullseye + - debian:bookworm steps: - uses: actions/checkout@v3 @@ -260,8 +326,9 @@ jobs: fail-fast: false matrix: distro: - - ubi9/ubi - - ubi8/ubi + - amazonlinux:2023 + - registry.access.redhat.com/ubi8/ubi + - registry.access.redhat.com/ubi9/ubi steps: - uses: actions/checkout@v3 @@ -276,7 +343,7 @@ jobs: - name: generate dockerfile run: | - echo 'FROM registry.access.redhat.com/${{ matrix.distro }} + echo 'FROM ${{ matrix.distro }} COPY tyk-pump*.x86_64.rpm /tyk-pump.rpm RUN yum install --allowerasing -y curl RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.rpm.sh | bash && yum install -y tyk-pump-1.6.0-1 @@ -293,9 +360,11 @@ jobs: push: false smoke-tests: - if: startsWith(github.ref, 'refs/tags') needs: - goreleaser + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout runs-on: ubuntu-latest steps: @@ -303,10 +372,23 @@ jobs: with: fetch-depth: 1 - - name: Run tests + - uses: aws-actions/configure-aws-credentials@v2 + with: + role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk + role-session-name: cipush + aws-region: eu-central-1 + + - id: ecr + uses: aws-actions/amazon-ecr-login@v1 + with: + mask-password: 'true' + + - name: Run ci/tests shell: bash env: GITHUB_TAG: ${{ github.ref }} + GATEWAY_IMAGE: ${{ steps.ecr.outputs.registry }}/tyk:${{ github.sha }} + PLUGIN_COMPILER_IMAGE: ${{ steps.ecr.outputs.registry }}/tyk-plugin-compiler:sha-${{ github.sha }} run: | set -eaxo pipefail if [ ! -d smoke-tests ]; then @@ -335,3 +417,11 @@ jobs: fi done + + sbom: + needs: goreleaser + uses: TykTechnologies/github-actions/.github/workflows/sbom.yaml@main + secrets: + DEPDASH_URL: ${{ secrets.DEPDASH_URL }} + DEPDASH_KEY: ${{ secrets.DEPDASH_KEY }} + ORG_GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }} diff --git a/ci/Dockerfile.std b/ci/Dockerfile.std index cce463d35..4c0617c7d 100644 --- a/ci/Dockerfile.std +++ b/ci/Dockerfile.std @@ -1,8 +1,7 @@ - # Generated by: gromit policy -# Generated on: Wed Jul 26 08:25:03 UTC 2023 +# Generated on: Thu Oct 26 05:43:34 UTC 2023 -FROM debian:bullseye-slim +FROM debian:bookworm-slim ARG TARGETARCH ENV DEBIAN_FRONTEND=noninteractive @@ -10,9 +9,8 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get dist-upgrade -y ca-certificates - # Remove some things to decrease CVE surface -RUN apt-get remove -y --allow-remove-essential --auto-remove curl libtiff5 ncurses-base \ +RUN dpkg --purge --force-remove-essential curl libtiff5 ncurses-base \ && rm /usr/bin/passwd && rm /usr/sbin/adduser # Clean up caches, unwanted .a and .o files diff --git a/ci/aws/byol.pkr.hcl b/ci/aws/byol.pkr.hcl index 167d9afdf..a09d67393 100644 --- a/ci/aws/byol.pkr.hcl +++ b/ci/aws/byol.pkr.hcl @@ -1,6 +1,6 @@ # Generated by: gromit policy -# Generated on: Wed Jul 26 08:25:03 UTC 2023 +# Generated on: Thu Oct 26 05:43:34 UTC 2023 packer { required_plugins { diff --git a/ci/goreleaser/goreleaser.yml b/ci/goreleaser/goreleaser.yml index d7666a2b4..be1e8d2a0 100644 --- a/ci/goreleaser/goreleaser.yml +++ b/ci/goreleaser/goreleaser.yml @@ -1,5 +1,5 @@ # Generated by: gromit policy -# Generated on: Wed Jul 26 08:25:03 UTC 2023 +# Generated on: Thu Oct 26 05:43:34 UTC 2023 # Check the documentation at http://goreleaser.com # This project needs CGO_ENABLED=1 and the cross-compiler toolchains for @@ -14,6 +14,7 @@ builds: goarch: - amd64 - arm64 + - s390x binary: tyk-pump dockers: @@ -21,8 +22,8 @@ dockers: - ids: - std image_templates: - - "tykio/tyk-pump-docker-pub:{{ .Tag }}-amd64" - - "docker.tyk.io/tyk-pump/tyk-pump:{{ .Tag }}-amd64" + - "tykio/tyk-pump-docker-pub:{{.Tag}}-amd64" + - "docker.tyk.io/tyk-pump/tyk-pump:{{.Tag}}-amd64" build_flag_templates: - "--build-arg=PORTS=80" - "--platform=linux/amd64" @@ -43,8 +44,8 @@ dockers: - ids: - std image_templates: - - "tykio/tyk-pump-docker-pub:{{ .Tag }}-arm64" - - "docker.tyk.io/tyk-pump/tyk-pump:{{ .Tag }}-arm64" + - "tykio/tyk-pump-docker-pub:{{.Tag}}-arm64" + - "docker.tyk.io/tyk-pump/tyk-pump:{{.Tag}}-arm64" build_flag_templates: - "--build-arg=PORTS=80" - "--platform=linux/arm64" @@ -128,6 +129,11 @@ publishers: cmd: /pc.sh {{ .ArtifactPath }} +# This disables archives +archives: + - format: binary + allow_different_binary_count: true + checksum: disable: true diff --git a/ci/install/before_install.sh b/ci/install/before_install.sh index 6b30c85a1..114f6246e 100755 --- a/ci/install/before_install.sh +++ b/ci/install/before_install.sh @@ -1,7 +1,7 @@ #!/bin/bash # Generated by: gromit policy -# Generated on: Wed Jul 26 08:25:03 UTC 2023 +# Generated on: Thu Oct 26 05:43:34 UTC 2023 echo "Creating user and group..." GROUPNAME="tyk" diff --git a/ci/install/post_install.sh b/ci/install/post_install.sh index ea03d859c..60081f36c 100755 --- a/ci/install/post_install.sh +++ b/ci/install/post_install.sh @@ -2,7 +2,7 @@ # Generated by: gromit policy -# Generated on: Wed Jul 26 08:25:03 UTC 2023 +# Generated on: Thu Oct 26 05:43:34 UTC 2023 # If "True" the install directory ownership will be changed to "tyk:tyk" change_ownership="True" diff --git a/ci/install/post_remove.sh b/ci/install/post_remove.sh index 016299b1f..77ff9375e 100755 --- a/ci/install/post_remove.sh +++ b/ci/install/post_remove.sh @@ -1,7 +1,7 @@ #!/bin/sh # Generated by: gromit policy -# Generated on: Wed Jul 26 08:25:03 UTC 2023 +# Generated on: Thu Oct 26 05:43:34 UTC 2023 cleanRemove() { diff --git a/ci/install/post_trans.sh b/ci/install/post_trans.sh index 0ccb92d6d..281d2675c 100644 --- a/ci/install/post_trans.sh +++ b/ci/install/post_trans.sh @@ -1,7 +1,7 @@ #!/bin/sh # Generated by: gromit policy -# Generated on: Wed Jul 26 08:25:03 UTC 2023 +# Generated on: Thu Oct 26 05:43:34 UTC 2023 if command -V systemctl >/dev/null 2>&1; then if [ ! -f /lib/systemd/system/tyk-pump.service ]; then