diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1b9168d5b..087586507 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,5 @@ # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Sun Dec 31 17:42:11 UTC 2023 # Distribution channels covered by this workflow # - Ubuntu and Debian @@ -141,7 +141,7 @@ jobs: type=semver,pattern=v{{version}},prefix=v - name: CI push - if: ${{ matrix.golang_cross == '1.19-bullseye' }} + if: ${{ matrix.golang_cross == '1.19-bullseye' }} shell: bash env: t: ${{ steps.metadata.outputs.tags }} @@ -173,17 +173,46 @@ jobs: path: | dist/*.rpm !dist/*PAYG*.rpm - api-tests: + + test-controller: needs: goreleaser runs-on: ubuntu-latest + container: tykio/gromit:v1.6 + outputs: + conf: ${{ steps.params.outputs.conf }} + db: ${{ steps.params.outputs.db }} + pump: ${{ steps.params.outputs.pump }} + sink: ${{ steps.params.outputs.sink }} + gd_tag: ${{ steps.params.outputs.gd_tag }} + versions: ${{ steps.params.outputs.versions }} + + steps: + - name: set params + id: params + env: + REPO: ${{ github.repository }} + # Cover pull_request_target too + BASE_REF: startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref + TAGS: ${{ needs.goreleaser.outputs.tags }} + IS_PR: ${{startsWith(github.event_name, 'pull_request') && 'yes' }} + IS_LTSPR: ${{(startsWith(github.event_name, 'pull_request') && endsWith(github.base_ref, '-lts')) && 'yes' }} + IS_TAG: ${{startsWith(github.ref, 'refs/tags') && 'yes' }} + run: gromit policy controller --debug | tee -a "$GITHUB_OUTPUT" + + + api-tests: + needs: test-controller + runs-on: ubuntu-latest permissions: - id-token: write # This is required for requesting the JWT + id-token: write # This is required for requesting the Github JWT contents: read # This is required for actions/checkout strategy: fail-fast: false matrix: - conf: [ "sha256", "murmur64" ] - db: [ "mongo44", "postgres15" ] + conf: ${{ fromJson(needs.test-controller.outputs.conf) }} + db: ${{ fromJson(needs.test-controller.outputs.db) }} + + sink: ${{ fromJson(needs.test-controller.outputs.sink) }} include: - db: postgres15 markers: "and not sql" @@ -222,33 +251,23 @@ jobs: working-directory: auto id: env_up env: - t: ${{ needs.goreleaser.outputs.tags }} pull_policy: 'if_not_present' - # default tag to use - default_image_tag: ${{ startsWith(github.ref_name, 'release-') && github.ref_name || 'master' }} GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }} TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }} TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }} + ECR: ${{ steps.ecr.outputs.registry }} + VERSIONS: ${{ needs.test-controller.outputs.versions}} run: | - echo CI tags: $t - tags=($t) - echo First tag: ${tags[0]} # Start customising the env - echo "registry=${{ steps.ecr.outputs.registry }} - tyk_image=\${registry}/tyk:${default_image_tag} - tyk_analytics_image=\${registry}/tyk-analytics:${default_image_tag} - tyk_sink_image=\${registry}/tyk-sink:${default_image_tag} - tyk_pump_image=\${registry}/tyk-pump:${default_image_tag} - # override default above with just built tag - tyk_pump_image=${tags[0]} - # alfa and beta have to come after the override - tyk_alfa_image=\${tyk_image} - tyk_beta_image=\${tyk_image} + echo "ECR=${ECR} + $VERSIONS + tyk_sink_image=${{matrix.sink}} # base dir for config files confs_dir=./pro-ha # pick database to use env_file=local-${{ matrix.db }}.env " > versions.env + printf '::group::versions\n%s\n::endgroup::' "$(cat versions.env)" # Add Tyk component config variations to $env_file cat confs/${{ matrix.conf }}.env >> local-${{ matrix.db }}.env # bring up env, the project name is important @@ -260,15 +279,15 @@ jobs: working-directory: auto id: test_execution run: | - echo "## API tests ${{ matrix.db }} ${{ matrix.conf }}" >> $GITHUB_STEP_SUMMARY + echo "### API tests ${{ matrix.db }} ${{ matrix.conf }}" >> $GITHUB_STEP_SUMMARY if docker run --rm --network auto_default --env-file pytest.env -v ${{ github.workspace }}/reports:/app/reports \ - ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:${{ startsWith(github.ref_name, 'release-') && github.ref_name || 'master' }} \ - pytest -c pytest_ci.ini --ci -m "not local and not dind ${{ matrix.markers }}" > tests.out; then + ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:${{ needs.test-controller.outputs.gd_tag }} \ + pytest -c pytest_ci.ini --ci -m "not local and not dind ${{ matrix.markers }}" | tee tests.out; then echo "All tests passed!" >> $GITHUB_STEP_SUMMARY else echo "::error title=API tests ${{ matrix.db }} ${{ matrix.conf }}::Test execution failed" cat tests.out >> $GITHUB_STEP_SUMMARY - docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env logs tyk tyk-analytics + docker logs tyk tyk-analytics exit 1 fi @@ -420,6 +439,7 @@ jobs: id-token: write # This is required for requesting the JWT contents: read # This is required for actions/checkout uses: ./.github/workflows/release-tests.yml + secrets: inherit sbom: diff --git a/ci/Dockerfile.std b/ci/Dockerfile.std index 810cda963..94a0eb7cc 100644 --- a/ci/Dockerfile.std +++ b/ci/Dockerfile.std @@ -1,5 +1,6 @@ # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Sun Dec 31 17:42:11 UTC 2023 + FROM debian:bullseye-slim ARG TARGETARCH @@ -9,8 +10,8 @@ RUN apt-get update \ && apt-get dist-upgrade -y ca-certificates # Remove some things to decrease CVE surface -RUN dpkg --purge --force-remove-essential curl ncurses-base \ - && rm /usr/bin/passwd && rm /usr/sbin/adduser +RUN dpkg --purge --force-remove-essential curl ncurses-base || true +RUN rm -fv /usr/bin/passwd /usr/sbin/adduser || true # Clean up caches, unwanted .a and .o files RUN rm -rf /root/.cache \ diff --git a/ci/aws/byol.pkr.hcl b/ci/aws/byol.pkr.hcl deleted file mode 100644 index 71dc51a9f..000000000 --- a/ci/aws/byol.pkr.hcl +++ /dev/null @@ -1,101 +0,0 @@ - -# Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 - -packer { - required_plugins { - amazon = { - version = ">= 0.0.2" - source = "github.com/hashicorp/amazon" - } - } -} - -variable "flavour" { - description = "OS Flavour" - type = string -} - -variable "source_ami_owner" { - type = string -} - -variable "ami_search_string" { - type = string -} - -variable "region" { - type = string - default = "us-east-1" -} - -variable "version" { - type = string - default = "${env("VERSION")}" -} - -# Latest at this time -data "amazon-ami" "base-os" { - filters = { - architecture = "x86_64" - "block-device-mapping.volume-type" = "gp2" - name = "${var.ami_search_string}" - root-device-type = "ebs" - sriov-net-support = "simple" - virtualization-type = "hvm" - } - most_recent = true - owners = ["${var.source_ami_owner}"] -} - -# source blocks are generated from your builders; a source can be referenced in -# build blocks. A build block runs provisioner and post-processors on a -# source. Read the documentation for source blocks here: -# https://www.packer.io/docs/from-1.5/blocks/source -source "amazon-ebs" "byol" { - ami_name = "BYOL tyk-pump ${var.version} (${var.flavour})" - ena_support = true - force_delete_snapshot = true - force_deregister = true - instance_type = "t3.micro" - region = "${var.region}" - source_ami = data.amazon-ami.base-os.id - sriov_support = true - ssh_username = "ec2-user" - subnet_filter { - filters = { - "tag:Class" = "build" - } - most_free = true - random = false - } - tags = { - Component = "tyk-pump" - Flavour = "${var.flavour}" - Product = "byol" - Version = "${var.version}" - } -} - -# a build block invokes sources and runs provisioning steps on them. The -# documentation for build blocks can be found here: -# https://www.packer.io/docs/from-1.5/blocks/build -build { - sources = ["source.amazon-ebs.byol"] - provisioner "file" { - destination = "/tmp/semver.sh" - source = "utils/semver.sh" - } - provisioner "file" { - destination = "/tmp/tyk-pump.rpm" - sources = fileset(".", "rpm/*x86_64.rpm") - } - provisioner "file" { - destination = "/tmp/10-run-tyk.conf" - source = "utils/10-run-tyk.conf" - } - provisioner "shell" { - environment_vars = ["VERSION=${var.version}" ] - script = "byol/install-tyk-pump.sh" - } -} diff --git a/ci/goreleaser/goreleaser.yml b/ci/goreleaser/goreleaser.yml index a581a0595..62071ef8d 100644 --- a/ci/goreleaser/goreleaser.yml +++ b/ci/goreleaser/goreleaser.yml @@ -1,5 +1,5 @@ # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Sun Dec 31 17:42:11 UTC 2023 # Check the documentation at http://goreleaser.com # This project needs CGO_ENABLED=1 and the cross-compiler toolchains for diff --git a/ci/install/before_install.sh b/ci/install/before_install.sh index 1d9d378f5..9770abbde 100755 --- a/ci/install/before_install.sh +++ b/ci/install/before_install.sh @@ -1,7 +1,7 @@ #!/bin/bash # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Sun Dec 31 17:42:11 UTC 2023 echo "Creating user and group..." GROUPNAME="tyk" diff --git a/ci/install/post_install.sh b/ci/install/post_install.sh index 4ff5aa316..67aa90bca 100755 --- a/ci/install/post_install.sh +++ b/ci/install/post_install.sh @@ -2,7 +2,7 @@ # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Sun Dec 31 17:42:11 UTC 2023 # If "True" the install directory ownership will be changed to "tyk:tyk" change_ownership="True" diff --git a/ci/install/post_remove.sh b/ci/install/post_remove.sh index 76f83c4fe..58bff35d1 100755 --- a/ci/install/post_remove.sh +++ b/ci/install/post_remove.sh @@ -1,7 +1,7 @@ #!/bin/sh # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Sun Dec 31 17:42:11 UTC 2023 cleanRemove() { diff --git a/ci/install/post_trans.sh b/ci/install/post_trans.sh index de77ca59c..4b51ebdec 100644 --- a/ci/install/post_trans.sh +++ b/ci/install/post_trans.sh @@ -1,7 +1,7 @@ #!/bin/sh # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Sun Dec 31 17:42:11 UTC 2023 if command -V systemctl >/dev/null 2>&1; then if [ ! -f /lib/systemd/system/tyk-pump.service ]; then