From dd87e5d5b4f7c15181cf1abb3ca4fe865fa167c1 Mon Sep 17 00:00:00 2001 From: Gromit Date: Thu, 18 Jan 2024 08:31:08 +0000 Subject: [PATCH] Auto generated from templates by gromit --- .github/workflows/del-env.yml | 21 -- .github/workflows/release.yml | 222 ++++++++------------ .github/workflows/sync-automation.yml | 99 --------- ci/Dockerfile.slim | 18 -- ci/Dockerfile.std | 7 +- ci/aws/al2.vars.json | 5 - ci/aws/byol.pkr.hcl | 101 ---------- ci/aws/byol/install-tyk-pump.sh | 9 - ci/aws/rhel.vars.json | 5 - ci/aws/utils/10-run-tyk.conf | 3 - ci/aws/utils/semver.sh | 278 -------------------------- ci/bin/pc.sh | 74 +++++++ ci/bin/unlock-agent.sh | 3 + ci/goreleaser/goreleaser.yml | 16 +- ci/image/Dockerfile | 24 --- ci/install/before_install.sh | 2 +- ci/install/post_install.sh | 2 +- ci/install/post_remove.sh | 2 +- ci/install/post_trans.sh | 2 +- 19 files changed, 170 insertions(+), 723 deletions(-) delete mode 100644 .github/workflows/del-env.yml delete mode 100644 .github/workflows/sync-automation.yml delete mode 100644 ci/Dockerfile.slim delete mode 100644 ci/aws/al2.vars.json delete mode 100644 ci/aws/byol.pkr.hcl delete mode 100755 ci/aws/byol/install-tyk-pump.sh delete mode 100644 ci/aws/rhel.vars.json delete mode 100644 ci/aws/utils/10-run-tyk.conf delete mode 100755 ci/aws/utils/semver.sh create mode 100755 ci/bin/pc.sh delete mode 100644 ci/image/Dockerfile mode change 100644 => 100755 ci/install/post_trans.sh diff --git a/.github/workflows/del-env.yml b/.github/workflows/del-env.yml deleted file mode 100644 index 99549e193..000000000 --- a/.github/workflows/del-env.yml +++ /dev/null @@ -1,21 +0,0 @@ - -# Generated by: gromit policy -# Generated on: Wed May 10 06:24:08 UTC 2023 - -name: Retiring dev env - -on: - delete: - branches: - - feature/* - - integration/* - -jobs: - retire: - runs-on: ubuntu-latest - - steps: - - name: Tell gromit about deleted branch - run: | - curl -fsSL -H "Authorization: ${{secrets.GROMIT_TOKEN}}" "https://domu-kun.cloud.tyk.io/gromit/env/${GITHUB_REF##*/}" \ - -X DELETE diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2a817c081..5b01081f0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,5 +1,5 @@ # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Thu Jan 18 08:31:08 UTC 2024 # Distribution channels covered by this workflow # - Ubuntu and Debian @@ -9,11 +9,10 @@ # - Cloudsmith name: Release - on: # Trigger release every monday at midnight for master CI images schedule: - - cron: "0 0 * * 1" + - cron: "0 0 * * 1" pull_request: push: branches: @@ -21,51 +20,35 @@ on: - release-** tags: - 'v*' - env: GOPRIVATE: github.com/TykTechnologies - jobs: goreleaser: name: '${{ matrix.golang_cross }}' runs-on: ubuntu-latest - container: 'tykio/golang-cross:${{ matrix.golang_cross }}' permissions: - id-token: write # AWS OIDC JWT - contents: read # actions/checkout + id-token: write # AWS OIDC JWT + contents: read # actions/checkout strategy: fail-fast: false matrix: golang_cross: - - 1.21-bullseye - include: + - 1.21-bullseye + include: - golang_cross: 1.21-bullseye goreleaser: 'ci/goreleaser/goreleaser.yml' - cgo: 0 + cgo: 0 rpmvers: 'el/7 el/8 el/9 amazon/2 amazon/2023' - debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye debian/bookworm' + debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye debian/bookworm' outputs: tags: ${{ steps.metadata.outputs.tags }} - steps: - - name: Fix private module deps - env: - TOKEN: '${{ secrets.ORG_GH_TOKEN }}' - run: > - git config --global url."https://${TOKEN}@github.com".insteadOf "https://github.com" - - name: Checkout of tyk-pump uses: actions/checkout@v4 with: fetch-depth: 1 - - - name: "Add Git safe.directory" - run: git config --global --add safe.directory $GITHUB_WORKSPACE - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - name: Login to DockerHub if: startsWith(github.ref, 'refs/tags') uses: docker/login-action@v3 @@ -79,17 +62,6 @@ jobs: registry: docker.tyk.io username: ${{ secrets.CLOUDSMITH_USERNAME }} password: ${{ secrets.CLOUDSMITH_API_KEY }} - - - name: Unlock agent - env: - NFPM_STD_PASSPHRASE: ${{ secrets.SIGNING_KEY_PASSPHRASE }} - GPG_FINGERPRINT: 12B5D62C28F57592D1575BD51ED14C59E37DAC20 - PKG_SIGNING_KEY: ${{ secrets.SIGNING_KEY }} - run: - ci/bin/unlock-agent.sh - - - - uses: actions/cache@v3 with: path: | @@ -98,22 +70,36 @@ jobs: key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} - - - uses: goreleaser/goreleaser-action@v5 - with: - version: latest - args: release --clean -f ${{ matrix.goreleaser }} ${{ !startsWith(github.ref, 'refs/tags/') && ' --snapshot' || '' }} + - name: Build env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CGO_ENABLED: ${{ matrix.cgo }} NFPM_STD_PASSPHRASE: ${{ secrets.SIGNING_KEY_PASSPHRASE }} - NFPM_PAYG_PASSPHRASE: ${{ secrets.SIGNING_KEY_PASSPHRASE }} - GPG_FINGERPRINT: 12B5D62C28F57592D1575BD51ED14C59E37DAC20 PKG_SIGNING_KEY: ${{ secrets.SIGNING_KEY }} - GOLANG_CROSS: ${{ matrix.golang_cross }} - DEBVERS: ${{ matrix.debvers }} - RPMVERS: ${{ matrix.rpmvers }} PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }} + run: | + echo '#!/bin/sh + ci/bin/unlock-agent.sh + git config --global url."https://${TOKEN}@github.com".insteadOf "https://github.com" + git config --global --add safe.directory /go/src/github.com/TykTechnologies/tyk-pump + goreleaser release --clean -f ${{ matrix.goreleaser }} ${{ !startsWith(github.ref, 'refs/tags/') && ' --snapshot' || '' }}' | tee /tmp/build.sh + chmod +x /tmp/build.sh + docker run --rm --privileged -e GITHUB_TOKEN=${{ github.token }} \ + -e GOPRIVATE=github.com/TykTechnologies \ + -e GO111MODULE=on \ + -e DEBVERS='${{ matrix.debvers }}' \ + -e RPMVERS='${{ matrix.rpmvers }}' \ + -e CGO_ENABLED=${{ matrix.cgo }} \ + -e NFPM_STD_PASSPHRASE="$NFPM_STD_PASSPHRASE" \ + -e GPG_FINGERPRINT=12B5D62C28F57592D1575BD51ED14C59E37DAC20 \ + -e PKG_SIGNING_KEY="$PKG_SIGNING_KEY" \ + -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN \ + -v ${{github.workspace}}:/go/src/github.com/TykTechnologies/tyk-pump \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v ~/.docker/config.json:/root/.docker/config.json \ + -v ~/go/pkg/mod:/go/pkg/mod \ + -v /tmp/build.sh:/tmp/build.sh \ + -w /go/src/github.com/TykTechnologies/tyk-pump \ + tykio/golang-cross:${{ matrix.golang_cross }} /tmp/build.sh - uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk @@ -121,13 +107,11 @@ jobs: aws-region: eu-central-1 # Don't mask to pass it across job boundaries mask-aws-account-id: false - - uses: aws-actions/amazon-ecr-login@v2 id: ecr if: ${{ matrix.golang_cross == '1.21-bullseye' }} with: mask-password: 'true' - - name: Docker metadata for CI id: metadata if: ${{ matrix.golang_cross == '1.21-bullseye' }} @@ -142,7 +126,6 @@ jobs: type=sha,format=long type=semver,pattern=v{{major}}.{{minor}},prefix=v type=semver,pattern=v{{version}},prefix=v - - name: CI push if: ${{ matrix.golang_cross == '1.21-bullseye' }} shell: bash @@ -158,7 +141,6 @@ jobs: done docker manifest create ${tag} ${tag}-amd64 ${tag}-arm64 && docker manifest push ${tag} done - - uses: actions/upload-artifact@v3 if: ${{ matrix.golang_cross == '1.21-bullseye' }} with: @@ -167,7 +149,6 @@ jobs: path: | dist/*.deb !dist/*PAYG*.deb - - uses: actions/upload-artifact@v3 if: ${{ matrix.golang_cross == '1.21-bullseye' }} with: @@ -176,17 +157,40 @@ jobs: path: | dist/*.rpm !dist/*PAYG*.rpm - api-tests: + test-controller: needs: goreleaser runs-on: ubuntu-latest + container: tykio/gromit:v1.6 + outputs: + conf: ${{ steps.params.outputs.conf }} + db: ${{ steps.params.outputs.db }} + pump: ${{ steps.params.outputs.pump }} + sink: ${{ steps.params.outputs.sink }} + gd_tag: ${{ steps.params.outputs.gd_tag }} + versions: ${{ steps.params.outputs.versions }} + steps: + - name: set params + id: params + env: + REPO: ${{ github.repository }} + # Cover pull_request_target too + BASE_REF: ${{startsWith(github.event_name, 'pull_request') && github.base_ref || github.ref}} + TAGS: ${{ needs.goreleaser.outputs.tags }} + IS_PR: ${{startsWith(github.event_name, 'pull_request') && 'yes' }} + IS_TAG: ${{startsWith(github.ref, 'refs/tags') && 'yes' }} + run: gromit policy controller --loglevel debug | tee -a "$GITHUB_OUTPUT" + api-tests: + needs: test-controller + runs-on: ubuntu-latest permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout + id-token: write # This is required for requesting the Github JWT + contents: read # This is required for actions/checkout strategy: fail-fast: false matrix: - conf: [ "sha256", "murmur64" ] - db: [ "mongo44", "postgres15" ] + conf: ${{ fromJson(needs.test-controller.outputs.conf) }} + db: ${{ fromJson(needs.test-controller.outputs.db) }} + sink: ${{ fromJson(needs.test-controller.outputs.sink) }} include: - db: postgres15 markers: "and not sql" @@ -196,104 +200,67 @@ jobs: role-to-assume: arn:aws:iam::754489498669:role/ecr_rw_tyk role-session-name: cipush aws-region: eu-central-1 - - id: ecr uses: aws-actions/amazon-ecr-login@v2 with: mask-password: 'true' - mask-aws-account-id: 'true' - - # Only ${{ github.actor }} has access - # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys - + # Only ${{ github.actor }} has access + # See https://github.com/mxschmitt/action-tmate#use-registered-public-ssh-keys - name: Setup tmate session only in debug mode uses: mxschmitt/action-tmate@v3 if: ${{ runner.debug == '1' }} with: detached: true limit-access-to-actor: true - - name: fetch env from tyk-ci env: GH_TOKEN: ${{ github.token }} run: | gh release download v1.2 --repo github.com/TykTechnologies/tyk-ci -O env.tgz tar xzvf env.tgz - - name: env up shell: bash working-directory: auto id: env_up env: - t: ${{ needs.goreleaser.outputs.tags }} pull_policy: 'if_not_present' - # default tag to use - default_image_tag: ${{ startsWith(github.ref_name, 'release-') && github.ref_name || 'master' }} GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }} TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }} TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }} - run: | - echo CI tags: $t - tags=($t) - echo First tag: ${tags[0]} - # Start customising the env - echo "registry=${{ steps.ecr.outputs.registry }} - tyk_image=\${registry}/tyk:${default_image_tag} - tyk_analytics_image=\${registry}/tyk-analytics:${default_image_tag} - tyk_sink_image=\${registry}/tyk-sink:${default_image_tag} - tyk_pump_image=\${registry}/tyk-pump:${default_image_tag} - # override default above with just built tag - tyk_pump_image=${tags[0]} - # alfa and beta have to come after the override - tyk_alfa_image=\${tyk_image} - tyk_beta_image=\${tyk_image} - # base dir for config files - confs_dir=./pro-ha - # pick database to use - env_file=local-${{ matrix.db }}.env - " > versions.env - # Add Tyk component config variations to $env_file - cat confs/${{ matrix.conf }}.env >> local-${{ matrix.db }}.env - # bring up env, the project name is important - docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d - ./dash-bootstrap.sh http://localhost:3000 - docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d - + ECR: ${{ steps.ecr.outputs.registry }} + VERSIONS: ${{ needs.test-controller.outputs.versions}} + run: "echo \"ECR=${ECR}\n$VERSIONS\n\ntyk_sink_image=${{matrix.sink}} \n# base dir for config files\nconfs_dir=./pro-ha\n# pick database to use\nenv_file=local-${{ matrix.db }}.env\n\" > versions.env\nprintf '::group::versions\\n%s\\n::endgroup::' \"$(cat versions.env)\"\n# Add Tyk component config variations to $env_file\ncat confs/${{ matrix.conf }}.env >> local-${{ matrix.db }}.env\n# bring up env, the project name is important\ndocker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d\n./dash-bootstrap.sh http://localhost:3000\ndocker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d\n" - name: Run tests working-directory: auto id: test_execution run: | - echo "## API tests ${{ matrix.db }} ${{ matrix.conf }}" >> $GITHUB_STEP_SUMMARY + echo "### API tests ${{ matrix.db }} ${{ matrix.conf }}" >> $GITHUB_STEP_SUMMARY if docker run --rm --network auto_default --env-file pytest.env -v ${{ github.workspace }}/reports:/app/reports \ - ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:${{ startsWith(github.ref_name, 'release-') && github.ref_name || 'master' }} \ - pytest -c pytest_ci.ini --ci -m "not local and not dind ${{ matrix.markers }}" > tests.out; then + ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:${{ needs.test-controller.outputs.gd_tag }} \ + pytest -c pytest_ci.ini --ci -m "not local and not dind ${{ matrix.markers }}" | tee tests.out; then echo "All tests passed!" >> $GITHUB_STEP_SUMMARY else echo "::error title=API tests ${{ matrix.db }} ${{ matrix.conf }}::Test execution failed" cat tests.out >> $GITHUB_STEP_SUMMARY - docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env logs tyk tyk-analytics + docker logs tyk tyk-analytics exit 1 fi - - name: Archive Integration tests report - if: ${{ always() }} + if: ${{ always() }} uses: actions/upload-artifact@v3 with: - name: api-test-report - retention-days: 3 - path: ${{ github.workspace }}/reports - + name: api-test-report + retention-days: 3 + path: ${{ github.workspace }}/reports - name: Fetch commit author - if: ${{ steps.test_execution.outcome != 'success' && !github.event.pull_request.draft }} + if: ${{ steps.test_execution.outcome != 'success' && !github.event.pull_request.draft }} run: echo "GIT_USER_EMAIL=$(git show -s --format='%ae' HEAD)" >> $GITHUB_ENV - - name: Fetch slack user - if: ${{ steps.test_execution.outcome != 'success' && !github.event.pull_request.draft }} + if: ${{ steps.test_execution.outcome != 'success' && !github.event.pull_request.draft }} id: fetch_slack_user uses: TykTechnologies/github-actions/.github/actions/github-to-slack@main with: github_email: ${{ env.GIT_USER_EMAIL }} - - name: Notify slack if: ${{ steps.test_execution.outcome != 'success' && !github.event.pull_request.draft }} uses: rtCamp/action-slack-notify@v2 @@ -304,25 +271,16 @@ jobs: SLACK_USERNAME: API INTEGRATION TESTS SLACK_MESSAGE: "*Test*: ${{ matrix.db }}-${{ matrix.conf }}, *Author*: ${{ steps.fetch_slack_user.outputs.slack-user-name }}" SLACK_FOOTER: "" - - name: Comment on PR if: ${{ always() && !github.event.pull_request.draft }} uses: mshick/add-pr-comment@v2 with: - message: | - **API tests result - ${{ matrix.db }}-${{ matrix.conf }} env: ${{ steps.test_execution.outcome }}** ${{ env.STATUS }} - Branch used: ${{ github.ref }} - Commit: ${{ github.event.after }} ${{ github.event.commits[0].message }} - Triggered by: ${{ github.event_name }} (@${{ github.actor }}) - [Execution page](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) + message: "**API tests result - ${{ matrix.db }}-${{ matrix.conf }} env: ${{ steps.test_execution.outcome }}** ${{ env.STATUS }} \nBranch used: ${{ github.ref }}\nCommit: ${{ github.event.after }} ${{ github.event.commits[0].message }}\nTriggered by: ${{ github.event_name }} (@${{ github.actor }})\n[Execution page](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})\n" repo-token: ${{ secrets.ORG_GH_TOKEN }} message-id: ${{ matrix.db }}-${{ matrix.conf }} env: STATUS: "${{ steps.test_execution.outcome == 'success' && ':white_check_mark:' || ':no_entry_sign:' }}" - - upgrade-deb: - if: startsWith(github.ref, 'refs/tags') services: httpbin.org: image: kennethreitz/httpbin @@ -340,20 +298,15 @@ jobs: - ubuntu:jammy - debian:bullseye - debian:bookworm - steps: - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: actions/download-artifact@v3 with: name: deb - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - name: generate dockerfile run: | echo 'FROM ${{ matrix.distro }} @@ -363,7 +316,6 @@ jobs: RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.deb.sh | bash && apt-get install -y tyk-pump=1.6.0 RUN dpkg -i tyk-pump.deb ' > Dockerfile - - name: install on ${{ matrix.distro }} uses: docker/build-push-action@v5 with: @@ -371,9 +323,7 @@ jobs: platforms: linux/${{ matrix.arch }} file: Dockerfile push: false - upgrade-rpm: - if: startsWith(github.ref, 'refs/tags') services: httpbin.org: image: kennethreitz/httpbin @@ -386,45 +336,31 @@ jobs: - amazonlinux:2023 - registry.access.redhat.com/ubi8/ubi - registry.access.redhat.com/ubi9/ubi - steps: - uses: actions/checkout@v4 with: fetch-depth: 1 - - uses: actions/download-artifact@v3 with: name: rpm - - uses: docker/setup-buildx-action@v3 - - name: generate dockerfile run: | echo 'FROM ${{ matrix.distro }} + SHELL ["/bin/bash", "-c"] COPY tyk-pump*.x86_64.rpm /tyk-pump.rpm - RUN yum install --allowerasing -y curl + RUN command -v curl || yum install curl RUN curl -fsSL https://packagecloud.io/install/repositories/tyk/tyk-pump/script.rpm.sh | bash && yum install -y tyk-pump-1.6.0-1 RUN curl https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 -o tyk-pump.key && rpm --import tyk-pump.key RUN rpm --checksig tyk-pump.rpm RUN rpm -Uvh --force tyk-pump.rpm ' > Dockerfile - - name: install on ${{ matrix.distro }} uses: docker/build-push-action@v5 with: context: "." file: Dockerfile push: false - - release-tests: - needs: - - goreleaser - permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout - uses: ./.github/workflows/release-tests.yml - - sbom: needs: goreleaser uses: TykTechnologies/github-actions/.github/workflows/sbom.yaml@main diff --git a/.github/workflows/sync-automation.yml b/.github/workflows/sync-automation.yml deleted file mode 100644 index 12626ad6f..000000000 --- a/.github/workflows/sync-automation.yml +++ /dev/null @@ -1,99 +0,0 @@ -# Generated by: gromit policy -# Generated on: Wed May 10 11:44:13 UTC 2023 - -name: Sync automation - -on: - push: - branches: - - master - paths: - - ci/** - - .github/workflows/release.yml - - .github/dependabot.yml - -jobs: - sync: - runs-on: ubuntu-latest - container: tykio/gromit:v1.5 - - strategy: - fail-fast: false - matrix: - branch: - - release-1.7 - - steps: - - uses: actions/checkout@v3 - with: - ref: ${{matrix.branch}} - token: ${{ secrets.ORG_GH_TOKEN }} - - - name: sync ${{matrix.branch}} from master - id: sync-changes - run: | - git config --global --add safe.directory "$GITHUB_WORKSPACE" - git config --local user.email "policy@gromit" - git config --local user.name "Bender" - git fetch origin ${{ matrix.branch }} - git fetch origin master:master - prbranch=sync/auto/${{ matrix.branch }}/${{ github.run_id }} - git checkout -b $prbranch - rm -rf ci - git restore --source master -- ci - rm -f .github/workflows/release.yml - git restore --source master -- .github/workflows/release.yml - rm -f .github/dependabot.yml - git restore --source master -- .github/dependabot.yml - git add -A - echo "prbranch=$prbranch" >> $GITHUB_OUTPUT - echo "::debug::Commit ${{ github.sha }} syncd for ${{matrix.branch}}" - echo "::debug::Generate releng bundle using latest gromit templates for ${{matrix.branch}}" - # get the tarball for the latest gromit master - it will have the latest version of - # the templates. - mkdir /tmp/gromit-src && gh api -H "Accept: application/vnd.github+json" /repos/TykTechnologies/gromit/tarball/master | tar --strip-components 1 -C /tmp/gromit-src -xzf - - gromit bundle gen --branch ${{ matrix.branch }} --bundle /tmp/gromit-src/policy/templates/releng --repo tyk-pump . - git add -A && git commit -m "[CI]: Syncing ${{ matrix.branch }}" - gromit git push . tyk-pump $prbranch --branch $prbranch - env: - GITHUB_TOKEN: ${{ secrets.ORG_GH_TOKEN }} - # Needed for gh CLI - GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }} - - - name: Create PR from the branch. - id: create-pr - uses: actions/github-script@v6 - with: - github-token: ${{ secrets.ORG_GH_TOKEN }} - result-encoding: string - script: | - const msg = ${{ toJSON(github.event.head_commit.message) }}; - const title = `[CI] Sync ${context.repo.repo}:${{ matrix.branch }} ` + msg.split("\n")[0]; - const response = await github.rest.pulls.create({ - title: title, - owner: context.repo.owner, - repo: context.repo.repo, - head: '${{ steps.sync-changes.outputs.prbranch }}', - base: '${{ matrix.branch }}', - body: `PR auto generated by the CI Sync automation. - Picks the CI changes from the commit ${{ github.sha }} on master. - Please make any additional changes required before merging. - ### _Commit message body of the picked commit:_ - ${msg} - `}); - github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: response.data.number, - labels: ['sync-automation'] }); - return response.data.number - - - - name: Enable automerge for the created PR - id: enable-automerge - run: | - gh pr merge $PULL --auto --squash --subject "[CI] Sync automation: Syncing commits from master" --body "Picking CI changes from the commit $COMMIT" - env: - GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }} - PULL: ${{ steps.create-pr.outputs.result }} - COMMIT: ${{ github.sha }} diff --git a/ci/Dockerfile.slim b/ci/Dockerfile.slim deleted file mode 100644 index 83e59fe2e..000000000 --- a/ci/Dockerfile.slim +++ /dev/null @@ -1,18 +0,0 @@ - - -# Generated by: tyk-ci/wf-gen -# Generated on: Friday 04 March 2022 09:23:10 AM UTC - -# Generation commands: -# ./pr.zsh -repos tyk-pump -base master -branch exp/sync-test -base master -title Sync from latest releng templates -p -# m4 -E -DxREPO=tyk-pump - -FROM gcr.io/distroless/static-debian10 -WORKDIR /opt/tyk-pump -COPY tyk-pump . -COPY pump.example.conf tyk-pump.conf -ARG PORTS -EXPOSE $PORTS - -ENTRYPOINT ["/opt/tyk-pump/tyk-pump" ] -CMD [ "--conf=/opt/tyk-pump/pump.conf" ] diff --git a/ci/Dockerfile.std b/ci/Dockerfile.std index 810cda963..2f2e1cf62 100644 --- a/ci/Dockerfile.std +++ b/ci/Dockerfile.std @@ -1,5 +1,6 @@ # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Thu Jan 18 08:31:08 UTC 2024 + FROM debian:bullseye-slim ARG TARGETARCH @@ -9,8 +10,8 @@ RUN apt-get update \ && apt-get dist-upgrade -y ca-certificates # Remove some things to decrease CVE surface -RUN dpkg --purge --force-remove-essential curl ncurses-base \ - && rm /usr/bin/passwd && rm /usr/sbin/adduser +RUN dpkg --purge --force-remove-essential curl ncurses-base || true +RUN rm -fv /usr/bin/passwd /usr/sbin/adduser || true # Clean up caches, unwanted .a and .o files RUN rm -rf /root/.cache \ diff --git a/ci/aws/al2.vars.json b/ci/aws/al2.vars.json deleted file mode 100644 index 97cbb85cc..000000000 --- a/ci/aws/al2.vars.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "ami_search_string": "amzn2-ami-hvm*", - "flavour": "AWSLinux", - "source_ami_owner": "amazon" -} diff --git a/ci/aws/byol.pkr.hcl b/ci/aws/byol.pkr.hcl deleted file mode 100644 index 71dc51a9f..000000000 --- a/ci/aws/byol.pkr.hcl +++ /dev/null @@ -1,101 +0,0 @@ - -# Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 - -packer { - required_plugins { - amazon = { - version = ">= 0.0.2" - source = "github.com/hashicorp/amazon" - } - } -} - -variable "flavour" { - description = "OS Flavour" - type = string -} - -variable "source_ami_owner" { - type = string -} - -variable "ami_search_string" { - type = string -} - -variable "region" { - type = string - default = "us-east-1" -} - -variable "version" { - type = string - default = "${env("VERSION")}" -} - -# Latest at this time -data "amazon-ami" "base-os" { - filters = { - architecture = "x86_64" - "block-device-mapping.volume-type" = "gp2" - name = "${var.ami_search_string}" - root-device-type = "ebs" - sriov-net-support = "simple" - virtualization-type = "hvm" - } - most_recent = true - owners = ["${var.source_ami_owner}"] -} - -# source blocks are generated from your builders; a source can be referenced in -# build blocks. A build block runs provisioner and post-processors on a -# source. Read the documentation for source blocks here: -# https://www.packer.io/docs/from-1.5/blocks/source -source "amazon-ebs" "byol" { - ami_name = "BYOL tyk-pump ${var.version} (${var.flavour})" - ena_support = true - force_delete_snapshot = true - force_deregister = true - instance_type = "t3.micro" - region = "${var.region}" - source_ami = data.amazon-ami.base-os.id - sriov_support = true - ssh_username = "ec2-user" - subnet_filter { - filters = { - "tag:Class" = "build" - } - most_free = true - random = false - } - tags = { - Component = "tyk-pump" - Flavour = "${var.flavour}" - Product = "byol" - Version = "${var.version}" - } -} - -# a build block invokes sources and runs provisioning steps on them. The -# documentation for build blocks can be found here: -# https://www.packer.io/docs/from-1.5/blocks/build -build { - sources = ["source.amazon-ebs.byol"] - provisioner "file" { - destination = "/tmp/semver.sh" - source = "utils/semver.sh" - } - provisioner "file" { - destination = "/tmp/tyk-pump.rpm" - sources = fileset(".", "rpm/*x86_64.rpm") - } - provisioner "file" { - destination = "/tmp/10-run-tyk.conf" - source = "utils/10-run-tyk.conf" - } - provisioner "shell" { - environment_vars = ["VERSION=${var.version}" ] - script = "byol/install-tyk-pump.sh" - } -} diff --git a/ci/aws/byol/install-tyk-pump.sh b/ci/aws/byol/install-tyk-pump.sh deleted file mode 100755 index 30130972b..000000000 --- a/ci/aws/byol/install-tyk-pump.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh -x - -curl -s https://packagecloud.io/install/repositories/tyk/tyk-pump/script.rpm.sh | sudo bash -sudo yum install -y pygpgme yum-utils wget - -sudo rpm -ivh /tmp/tyk-pump.rpm || exit 1 - -sudo rm -f /home/ec2-user/.ssh/authorized_keys -sudo rm -f /root/.ssh/authorized_keys diff --git a/ci/aws/rhel.vars.json b/ci/aws/rhel.vars.json deleted file mode 100644 index 7ec747355..000000000 --- a/ci/aws/rhel.vars.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "ami_search_string": "RHEL-7.?*GA*", - "flavour": "RedHat", - "source_ami_owner": "309956199498" -} diff --git a/ci/aws/utils/10-run-tyk.conf b/ci/aws/utils/10-run-tyk.conf deleted file mode 100644 index 91a833b12..000000000 --- a/ci/aws/utils/10-run-tyk.conf +++ /dev/null @@ -1,3 +0,0 @@ -[Service] -User=tyk -Group=tyk \ No newline at end of file diff --git a/ci/aws/utils/semver.sh b/ci/aws/utils/semver.sh deleted file mode 100755 index 5c5ac5a23..000000000 --- a/ci/aws/utils/semver.sh +++ /dev/null @@ -1,278 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -o nounset -o pipefail - -NAT='0|[1-9][0-9]*' -ALPHANUM='[0-9]*[A-Za-z-][0-9A-Za-z-]*' -IDENT="$NAT|$ALPHANUM" -FIELD='[0-9A-Za-z-]+' - -SEMVER_REGEX="\ -^[vV]?\ -($NAT)\\.($NAT)\\.($NAT)\ -(\\-(${IDENT})(\\.(${IDENT}))*)?\ -(\\+${FIELD}(\\.${FIELD})*)?$" - -PROG=semver -PROG_VERSION="3.0.0" - -USAGE="\ -Usage: - $PROG bump (major|minor|patch|release|prerel |build ) - $PROG compare - $PROG get (major|minor|patch|release|prerel|build) - $PROG --help - $PROG --version - -Arguments: - A version must match the following regular expression: - \"${SEMVER_REGEX}\" - In English: - -- The version must match X.Y.Z[-PRERELEASE][+BUILD] - where X, Y and Z are non-negative integers. - -- PRERELEASE is a dot separated sequence of non-negative integers and/or - identifiers composed of alphanumeric characters and hyphens (with - at least one non-digit). Numeric identifiers must not have leading - zeros. A hyphen (\"-\") introduces this optional part. - -- BUILD is a dot separated sequence of identifiers composed of alphanumeric - characters and hyphens. A plus (\"+\") introduces this optional part. - - See definition. - - A string as defined by PRERELEASE above. - - A string as defined by BUILD above. - -Options: - -v, --version Print the version of this tool. - -h, --help Print this help message. - -Commands: - bump Bump by one of major, minor, patch; zeroing or removing - subsequent parts. \"bump prerel\" sets the PRERELEASE part and - removes any BUILD part. \"bump build\" sets the BUILD part. - \"bump release\" removes any PRERELEASE or BUILD parts. - The bumped version is written to stdout. - - compare Compare with , output to stdout the - following values: -1 if is newer, 0 if equal, 1 if - older. The BUILD part is not used in comparisons. - - get Extract given part of , where part is one of major, minor, - patch, prerel, build, or release. - -See also: - https://semver.org -- Semantic Versioning 2.0.0" - -function error { - echo -e "$1" >&2 - exit 1 -} - -function usage-help { - error "$USAGE" -} - -function usage-version { - echo -e "${PROG}: $PROG_VERSION" - exit 0 -} - -function validate-version { - local version=$1 - if [[ "$version" =~ $SEMVER_REGEX ]]; then - # if a second argument is passed, store the result in var named by $2 - if [ "$#" -eq "2" ]; then - local major=${BASH_REMATCH[1]} - local minor=${BASH_REMATCH[2]} - local patch=${BASH_REMATCH[3]} - local prere=${BASH_REMATCH[4]} - local build=${BASH_REMATCH[8]} - eval "$2=(\"$major\" \"$minor\" \"$patch\" \"$prere\" \"$build\")" - else - echo "$version" - fi - else - error "version $version does not match the semver scheme 'X.Y.Z(-PRERELEASE)(+BUILD)'. See help for more information." - fi -} - -function is-nat { - [[ "$1" =~ ^($NAT)$ ]] -} - -function is-null { - [ -z "$1" ] -} - -function order-nat { - [ "$1" -lt "$2" ] && { echo -1 ; return ; } - [ "$1" -gt "$2" ] && { echo 1 ; return ; } - echo 0 -} - -function order-string { - [[ $1 < $2 ]] && { echo -1 ; return ; } - [[ $1 > $2 ]] && { echo 1 ; return ; } - echo 0 -} - -# given two (named) arrays containing NAT and/or ALPHANUM fields, compare them -# one by one according to semver 2.0.0 spec. Return -1, 0, 1 if left array ($1) -# is less-than, equal, or greater-than the right array ($2). The longer array -# is considered greater-than the shorter if the shorter is a prefix of the longer. -# -function compare-fields { - local l="$1[@]" - local r="$2[@]" - local leftfield=( "${!l}" ) - local rightfield=( "${!r}" ) - local left - local right - - local i=$(( -1 )) - local order=$(( 0 )) - - while true - do - [ $order -ne 0 ] && { echo $order ; return ; } - - : $(( i++ )) - left="${leftfield[$i]}" - right="${rightfield[$i]}" - - is-null "$left" && is-null "$right" && { echo 0 ; return ; } - is-null "$left" && { echo -1 ; return ; } - is-null "$right" && { echo 1 ; return ; } - - is-nat "$left" && is-nat "$right" && { order=$(order-nat "$left" "$right") ; continue ; } - is-nat "$left" && { echo -1 ; return ; } - is-nat "$right" && { echo 1 ; return ; } - { order=$(order-string "$left" "$right") ; continue ; } - done -} - -# shellcheck disable=SC2206 # checked by "validate"; ok to expand prerel id's into array -function compare-version { - local order - validate-version "$1" V - validate-version "$2" V_ - - # compare major, minor, patch - - local left=( "${V[0]}" "${V[1]}" "${V[2]}" ) - local right=( "${V_[0]}" "${V_[1]}" "${V_[2]}" ) - - order=$(compare-fields left right) - [ "$order" -ne 0 ] && { echo "$order" ; return ; } - - # compare pre-release ids when M.m.p are equal - - local prerel="${V[3]:1}" - local prerel_="${V_[3]:1}" - local left=( ${prerel//./ } ) - local right=( ${prerel_//./ } ) - - # if left and right have no pre-release part, then left equals right - # if only one of left/right has pre-release part, that one is less than simple M.m.p - - [ -z "$prerel" ] && [ -z "$prerel_" ] && { echo 0 ; return ; } - [ -z "$prerel" ] && { echo 1 ; return ; } - [ -z "$prerel_" ] && { echo -1 ; return ; } - - # otherwise, compare the pre-release id's - - compare-fields left right -} - -function command-bump { - local new; local version; local sub_version; local command; - - case $# in - 2) case $1 in - major|minor|patch|release) command=$1; version=$2;; - *) usage-help;; - esac ;; - 3) case $1 in - prerel|build) command=$1; sub_version=$2 version=$3 ;; - *) usage-help;; - esac ;; - *) usage-help;; - esac - - validate-version "$version" parts - # shellcheck disable=SC2154 - local major="${parts[0]}" - local minor="${parts[1]}" - local patch="${parts[2]}" - local prere="${parts[3]}" - local build="${parts[4]}" - - case "$command" in - major) new="$((major + 1)).0.0";; - minor) new="${major}.$((minor + 1)).0";; - patch) new="${major}.${minor}.$((patch + 1))";; - release) new="${major}.${minor}.${patch}";; - prerel) new=$(validate-version "${major}.${minor}.${patch}-${sub_version}");; - build) new=$(validate-version "${major}.${minor}.${patch}${prere}+${sub_version}");; - *) usage-help ;; - esac - - echo "$new" - exit 0 -} - -function command-compare { - local v; local v_; - - case $# in - 2) v=$(validate-version "$1"); v_=$(validate-version "$2") ;; - *) usage-help ;; - esac - - set +u # need unset array element to evaluate to null - compare-version "$v" "$v_" - exit 0 -} - - -# shellcheck disable=SC2034 -function command-get { - local part version - - if [[ "$#" -ne "2" ]] || [[ -z "$1" ]] || [[ -z "$2" ]]; then - usage-help - exit 0 - fi - - part="$1" - version="$2" - - validate-version "$version" parts - local major="${parts[0]}" - local minor="${parts[1]}" - local patch="${parts[2]}" - local prerel="${parts[3]:1}" - local build="${parts[4]:1}" - local release="${major}.${minor}.${patch}" - - case "$part" in - major|minor|patch|release|prerel|build) echo "${!part}" ;; - *) usage-help ;; - esac - - exit 0 -} - -case $# in - 0) echo "Unknown command: $*"; usage-help;; -esac - -case $1 in - --help|-h) echo -e "$USAGE"; exit 0;; - --version|-v) usage-version ;; - bump) shift; command-bump "$@";; - get) shift; command-get "$@";; - compare) shift; command-compare "$@";; - *) echo "Unknown arguments: $*"; usage-help;; -esac diff --git a/ci/bin/pc.sh b/ci/bin/pc.sh new file mode 100755 index 000000000..c2400c747 --- /dev/null +++ b/ci/bin/pc.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Generated by: gromit policy +# Generated on: Thu Jan 18 08:31:08 UTC 2024 + +# goreleaser calls a custom publisher for each artefact packagecloud +# expects the distro version when pushing this script bridges both by +# choosing the appropriate list of distro versions from $DEBVERS and +# $RPMVERS +# $REPO, $DEBVERS and $RPMVERS are expected to be set by the +# user + +REQUIRED_VARS="PACKAGECLOUD_TOKEN REPO" + +usage() { + cat <" \ + --define "%__gpg /usr/bin/gpg" \ + --addsign $pkg + fi + ;; + *) + echo "Unknown package, not uploading" +esac + +for i in $vers; do + + [[ ! -s ${pkg} ]] && echo "File is empty or does not exists" && exit 1 + + # Yank packages first to enable tag re-use + packagecloud rm $REPO/$i $(basename $pkg) || true + packagecloud push $REPO/$i $pkg + +done diff --git a/ci/bin/unlock-agent.sh b/ci/bin/unlock-agent.sh index 6cadc55a7..729cac613 100755 --- a/ci/bin/unlock-agent.sh +++ b/ci/bin/unlock-agent.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +# Generated by: gromit policy +# Generated on: Thu Jan 18 08:31:08 UTC 2024 + # Get the GPG fingerprint with gpg --with-keygrip --list-secret-keys if [[ -z "${PKG_SIGNING_KEY}" || -z "${NFPM_STD_PASSPHRASE}" || -z "${GPG_FINGERPRINT}" ]]; then echo "No private key set, packages cannnot be signed. Set PKG_SIGNING_KEY, NFPM_STD_PASSPHRASE and GPG_FINGERPRINT" diff --git a/ci/goreleaser/goreleaser.yml b/ci/goreleaser/goreleaser.yml index a581a0595..d791dd8af 100644 --- a/ci/goreleaser/goreleaser.yml +++ b/ci/goreleaser/goreleaser.yml @@ -1,5 +1,5 @@ # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Thu Jan 18 08:31:08 UTC 2024 # Check the documentation at http://goreleaser.com # This project needs CGO_ENABLED=1 and the cross-compiler toolchains for @@ -19,7 +19,6 @@ builds: - arm64 - s390x binary: tyk-pump - dockers: # Build tykio/tyk-pump-docker-pub, docker.tyk.io/tyk-pump/tyk-pump (amd64) - ids: @@ -65,7 +64,6 @@ dockers: - "README.md" - "LICENSE.md" - "pump.example.conf" - docker_manifests: - name_template: tykio/tyk-pump-docker-pub:{{ .Tag }} image_templates: @@ -75,11 +73,14 @@ docker_manifests: image_templates: - tykio/tyk-pump-docker-pub:{{ .Tag }}-amd64 - tykio/tyk-pump-docker-pub:{{ .Tag }}-arm64 + - name_template: tykio/tyk-pump-docker-pub:v{{ .Major }}{{.Prerelease}} + image_templates: + - tykio/tyk-pump-docker-pub:{{ .Tag }}-amd64 + - tykio/tyk-pump-docker-pub:{{ .Tag }}-arm64 - name_template: docker.tyk.io/tyk-pump/tyk-pump:{{ .Tag }} image_templates: - docker.tyk.io/tyk-pump/tyk-pump:{{ .Tag }}-amd64 - docker.tyk.io/tyk-pump/tyk-pump:{{ .Tag }}-arm64 - nfpms: - id: std vendor: "Tyk Technologies Ltd" @@ -121,7 +122,6 @@ nfpms: signature: key_file: tyk.io.signing.key type: origin - publishers: - name: tyk-pump-unstable env: @@ -129,17 +129,13 @@ publishers: - REPO=tyk/tyk-pump-unstable - RPMVERS={{ .Env.RPMVERS }} - DEBVERS={{ .Env.DEBVERS }} - cmd: /pc.sh {{ .ArtifactPath }} - - + cmd: ./ci/bin/pc.sh {{ .ArtifactPath }} # This disables archives archives: - format: binary allow_different_binary_count: true - checksum: disable: true - release: disable: true github: diff --git a/ci/image/Dockerfile b/ci/image/Dockerfile deleted file mode 100644 index e530b238d..000000000 --- a/ci/image/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# Generated on: Fri 11 Dec 00:40:39 IST 2020 - -# Generated by: wf-gen from tyk-ci - -FROM debian:buster-slim - -RUN apt-get update \ - && apt-get dist-upgrade -y --no-install-recommends \ - wget jq curl ca-certificates apt-transport-https gnupg unzip - - -RUN apt-get autoremove -y - -ADD tyk-pump.tar.gz /opt/tyk-pump - -VOLUME ["/conf"] -WORKDIR /opt/tyk-pump - -ENTRYPOINT ["/opt/tyk-pump/tyk-pump" ] -CMD [ "--conf=/conf/tyk-pump/tyk-pump.conf" ] - -# Local Variables: -# mode: dockerfile -# End: diff --git a/ci/install/before_install.sh b/ci/install/before_install.sh index 1d9d378f5..c7f86bee4 100755 --- a/ci/install/before_install.sh +++ b/ci/install/before_install.sh @@ -1,7 +1,7 @@ #!/bin/bash # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Thu Jan 18 08:31:08 UTC 2024 echo "Creating user and group..." GROUPNAME="tyk" diff --git a/ci/install/post_install.sh b/ci/install/post_install.sh index 4ff5aa316..fa99f5631 100755 --- a/ci/install/post_install.sh +++ b/ci/install/post_install.sh @@ -2,7 +2,7 @@ # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Thu Jan 18 08:31:08 UTC 2024 # If "True" the install directory ownership will be changed to "tyk:tyk" change_ownership="True" diff --git a/ci/install/post_remove.sh b/ci/install/post_remove.sh index 76f83c4fe..4c9eb01a4 100755 --- a/ci/install/post_remove.sh +++ b/ci/install/post_remove.sh @@ -1,7 +1,7 @@ #!/bin/sh # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Thu Jan 18 08:31:08 UTC 2024 cleanRemove() { diff --git a/ci/install/post_trans.sh b/ci/install/post_trans.sh old mode 100644 new mode 100755 index de77ca59c..520732ada --- a/ci/install/post_trans.sh +++ b/ci/install/post_trans.sh @@ -1,7 +1,7 @@ #!/bin/sh # Generated by: gromit policy -# Generated on: Wed Nov 29 23:09:25 UTC 2023 +# Generated on: Thu Jan 18 08:31:08 UTC 2024 if command -V systemctl >/dev/null 2>&1; then if [ ! -f /lib/systemd/system/tyk-pump.service ]; then