Performance Test #216
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Test | |
| on: | |
| pull_request: | |
| schedule: | |
| # don't know the timezone but it's daily at least | |
| - cron: '0 7 * * *' | |
| env: | |
| terraform_version: '1.2.4' | |
| HAS_ACCESS_TO_GITHUB_TOKEN: ${{ github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') }} | |
| BUILD_ROOT: ${{ github.workspace }}/bazel-bin/build | |
| # only for pr | |
| GHA_CACHE: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| build-packages: | |
| name: Build dependencies | |
| runs-on: ubuntu-22.04 | |
| if: | | |
| github.event_name == 'schedule' || | |
| (github.event_name == 'pull_request' && startsWith(github.event.pull_request.title, 'perf(')) || | |
| (github.event_name == 'issue_comment' && github.event.action == 'created' && | |
| github.event.issue.pull_request && | |
| contains('["OWNER", "COLLABORATOR", "MEMBER"]', github.event.comment.author_association) && | |
| (startsWith(github.event.comment.body, '/perf') || startsWith(github.event.comment.body, '/flamegraph')) | |
| ) | |
| outputs: | |
| cache-key: ${{ steps.cache-key.outputs.cache-key }} | |
| steps: | |
| - name: Checkout Kong source code | |
| uses: actions/checkout@v4 | |
| - name: Generate cache key | |
| id: cache-key | |
| uses: ./.github/actions/build-cache-key | |
| with: | |
| prefix: perf | |
| - name: Lookup build cache | |
| id: cache-deps | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.BUILD_ROOT }} | |
| key: ${{ steps.cache-key.outputs.cache-key }} | |
| - name: Install packages | |
| if: steps.cache-deps.outputs.cache-hit != 'true' | |
| run: sudo apt update && sudo apt install libyaml-dev valgrind libprotobuf-dev | |
| - name: Build Kong | |
| if: steps.cache-deps.outputs.cache-hit != 'true' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| make build-kong | |
| BUILD_PREFIX=$BUILD_ROOT/kong-dev | |
| export PATH="$BUILD_PREFIX/bin:$BUILD_PREFIX/openresty/nginx/sbin:$BUILD_PREFIX/openresty/bin:$PATH" | |
| chmod +rw -R $BUILD_PREFIX | |
| nginx -V | |
| ldd $(which nginx) | |
| luarocks | |
| - name: Bazel Outputs | |
| uses: actions/upload-artifact@v3 | |
| if: failure() | |
| with: | |
| name: bazel-outputs | |
| path: | | |
| bazel-out/_tmp/actions | |
| retention-days: 3 | |
| - name: Build Dev Kong dependencies | |
| if: steps.cache-deps.outputs.cache-hit != 'true' | |
| run: | | |
| make install-dev-rocks | |
| perf: | |
| name: RPS, latency and flamegraphs | |
| runs-on: ubuntu-22.04 | |
| needs: build-packages | |
| permissions: | |
| # required to send comment of graphs and results in the PR | |
| pull-requests: write | |
| if: | | |
| github.event_name == 'schedule' || | |
| (github.event_name == 'pull_request' && startsWith(github.event.pull_request.title, 'perf(')) || | |
| (github.event_name == 'issue_comment' && github.event.action == 'created' && | |
| github.event.issue.pull_request && | |
| contains('["OWNER", "COLLABORATOR", "MEMBER"]', github.event.comment.author_association) && | |
| (startsWith(github.event.comment.body, '/perf') || startsWith(github.event.comment.body, '/flamegraph')) | |
| ) | |
| # perf test can only run one at a time per repo for now | |
| concurrency: | |
| group: perf-ce | |
| steps: | |
| # set up mutex across CE and EE to avoid resource race | |
| - name: Set up mutex | |
| uses: ben-z/gh-action-mutex@9709ba4d8596ad4f9f8bbe8e0f626ae249b1b3ac # v1.0-alpha-6 | |
| with: | |
| repository: "Kong/kong-perf-mutex-lock" | |
| branch: "gh-mutex" | |
| repo-token: ${{ secrets.PAT }} | |
| - name: Checkout Kong source code | |
| uses: actions/checkout@v4 | |
| with: | |
| # Fetch all history for all tags and branches | |
| fetch-depth: 0 | |
| - name: Load Cached Packages | |
| id: cache-deps | |
| if: env.GHA_CACHE == 'true' | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.BUILD_ROOT }} | |
| key: ${{ needs.build-packages.outputs.cache-key }} | |
| - name: Install performance test Dependencies | |
| run: | | |
| # in Kong repository | |
| sudo apt update && sudo apt install inkscape -y | |
| # terraform! | |
| wget https://releases.hashicorp.com/terraform/${{ env.terraform_version }}/terraform_${{ env.terraform_version }}_linux_amd64.zip | |
| unzip terraform_${{ env.terraform_version }}_linux_amd64.zip | |
| sudo mv terraform /usr/bin/ | |
| - name: Choose perf suites | |
| id: choose_perf | |
| run: | | |
| suites="$(printf '%s' "${{ github.event.comment.body }}" | awk '{print $1}')" | |
| tags="$(printf '%s' "${{ github.event.comment.body }}" | awk '{print $2}')" | |
| if [[ $suite == "/flamegraph" ]]; then | |
| suites="02-flamegraph" | |
| if [[ -z $tags ]]; then | |
| tags="simple" | |
| fi | |
| elif [[ $suite == "/perf" ]]; then | |
| suites="01-rps" | |
| if [[ -z $tags ]]; then | |
| tags="baseline,single_route" | |
| fi | |
| else | |
| # if not specified by comment, run both | |
| suites="01-rps 02-flamegraph" | |
| if [[ -z $tags ]]; then | |
| tags="baseline,single_route,simple" | |
| fi | |
| fi | |
| echo "suites=$suites" >> $GITHUB_OUTPUT | |
| echo "tags=$tags" >> $GITHUB_OUTPUT | |
| - uses: xt0rted/pull-request-comment-branch@d97294d304604fa98a2600a6e2f916a84b596dc7 # v1.4.1 | |
| id: comment-branch | |
| if: github.event_name == 'issue_comment' && github.event.action == 'created' | |
| - name: Find compared versions | |
| id: compare_versions | |
| run: | | |
| pr_ref=$(echo "${{ github.event.pull_request.base.ref }}") | |
| custom_vers="$(printf '%s' "${{ github.event.comment.body }}" | awk '{print $3}')" | |
| if [[ ! -z "${pr_ref}" ]]; then | |
| vers="git:${{ github.head_ref }},git:${pr_ref}" | |
| elif [[ ! -z "${custom_vers}" ]]; then | |
| vers="${custom_vers}" | |
| elif [[ ! -z "${{ github.event.comment.body }}" ]]; then | |
| vers="git:${{ steps.comment-branch.outputs.head_ref}},git:${{ steps.comment-branch.outputs.base_ref}}" | |
| else # is cron job/on master | |
| vers="git:master,git:origin/master~10,git:origin/master~50" | |
| fi | |
| echo $vers | |
| echo "vers=$vers" >> $GITHUB_OUTPUT | |
| - name: Run Tests | |
| env: | |
| PERF_TEST_VERSIONS: ${{ steps.compare_versions.outputs.vers }} | |
| PERF_TEST_DRIVER: terraform | |
| PERF_TEST_TERRAFORM_PROVIDER: bring-your-own | |
| PERF_TEST_BYO_KONG_IP: ${{ secrets.PERF_TEST_BYO_KONG_IP }} | |
| PERF_TEST_BYO_WORKER_IP: ${{ secrets.PERF_TEST_BYO_WORKER_IP }} | |
| PERF_TEST_BYO_SSH_USER: gha | |
| PERF_TEST_USE_DAILY_IMAGE: true | |
| PERF_TEST_DISABLE_EXEC_OUTPUT: true | |
| timeout-minutes: 180 | |
| run: | | |
| export PERF_TEST_BYO_SSH_KEY_PATH=$(pwd)/ssh_key | |
| echo "${{ secrets.PERF_TEST_BYO_SSH_KEY }}" > ${PERF_TEST_BYO_SSH_KEY_PATH} | |
| chmod 600 ${PERF_TEST_BYO_SSH_KEY_PATH} | |
| # setup tunnel for psql and admin port | |
| ssh -o StrictHostKeyChecking=no -o TCPKeepAlive=yes -o ServerAliveInterval=10 \ | |
| -o ExitOnForwardFailure=yes -o ConnectTimeout=5 \ | |
| -L 15432:localhost:5432 -L 39001:localhost:39001 \ | |
| -i ${PERF_TEST_BYO_SSH_KEY_PATH} \ | |
| ${PERF_TEST_BYO_SSH_USER}@${PERF_TEST_BYO_KONG_IP} tail -f /dev/null & | |
| sleep 5 | |
| sudo iptables -t nat -I OUTPUT -p tcp --dport 5432 -d ${PERF_TEST_BYO_KONG_IP} -j DNAT --to 127.0.0.1:15432 | |
| sudo iptables -t nat -I OUTPUT -p tcp --dport 39001 -d ${PERF_TEST_BYO_KONG_IP} -j DNAT --to 127.0.0.1:39001 | |
| make dev # required to install other dependencies like bin/grpcurl | |
| source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh | |
| for suite in ${{ steps.choose_perf.outputs.suites }}; do | |
| # Run each test individually, ngx.pipe doesn't like to be imported twice | |
| # maybe bin/busted --no-auto-insulate | |
| for f in $(find "spec/04-perf/$suite/" -type f); do | |
| bin/busted "$f" \ | |
| -t "${{ steps.choose_perf.outputs.tags }}" | |
| done | |
| done | |
| - name: Teardown | |
| # Note: by default each job has if: ${{ success() }} | |
| if: always() | |
| env: | |
| PERF_TEST_VERSIONS: git:${{ github.sha }} | |
| PERF_TEST_DRIVER: terraform | |
| PERF_TEST_TERRAFORM_PROVIDER: bring-your-own | |
| PERF_TEST_BYO_KONG_IP: ${{ secrets.PERF_TEST_BYO_KONG_IP }} | |
| PERF_TEST_BYO_WORKER_IP: ${{ secrets.PERF_TEST_BYO_WORKER_IP }} | |
| PERF_TEST_BYO_SSH_USER: gha | |
| PERF_TEST_TEARDOWN_ALL: true | |
| run: | | |
| export PERF_TEST_BYO_SSH_KEY_PATH=$(pwd)/ssh_key | |
| echo "${{ secrets.PERF_TEST_BYO_SSH_KEY }}" > ${PERF_TEST_BYO_SSH_KEY_PATH} | |
| make dev # required to install other dependencies like bin/grpcurl | |
| source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh | |
| bin/busted spec/04-perf/99-teardown/ | |
| rm -f ${PERF_TEST_BYO_SSH_KEY_PATH} | |
| - name: Generate high DPI graphs | |
| if: always() | |
| run: | | |
| for i in $(ls output/*.svg); do | |
| inkscape --export-area-drawing --export-png="${i%.*}.png" --export-dpi=300 -b FFFFFF $i | |
| done | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| cache: 'pip' | |
| - name: Generate plots | |
| if: always() | |
| run: | | |
| cwd=$(pwd) | |
| cd spec/helpers/perf/charts/ | |
| pip install -r requirements.txt | |
| for i in $(ls ${cwd}/output/*.data.json); do | |
| python ./charts.py $i -o "${cwd}/output/" | |
| done | |
| - name: Save results | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: perf-results | |
| path: | | |
| output/ | |
| !output/**/*.log | |
| retention-days: 31 | |
| - name: Save error logs | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: error_logs | |
| path: | | |
| output/**/*.log | |
| retention-days: 31 | |
| - name: Output | |
| if: always() | |
| id: output | |
| run: | | |
| if [[ "${{ steps.choose_perf.outputs.suites }}" =~ "02-flamegraph" ]]; then | |
| result="Please see Github Actions artifacts for flamegraphs. | |
| " | |
| fi | |
| result="${result}$(cat output/result.txt)" || true | |
| # https://github.community/t/set-output-truncates-multiline-strings/16852/2 | |
| result="${result//'%'/'%25'}" | |
| result="${result//$'\n'/'%0A'}" | |
| result="${result//$'\r'/'%0D'}" | |
| echo "result=$results" >> $GITHUB_OUTPUT | |
| - name: Upload charts | |
| if: always() | |
| id: charts | |
| uses: devicons/public-upload-to-imgur@352cf5f2805c692539a96cfe49a09669e6fca88e # v2.2.2 | |
| continue-on-error: true | |
| with: | |
| path: output/*.png | |
| client_id: ${{ secrets.PERF_TEST_IMGUR_CLIENT_ID }} | |
| - name: Comment | |
| if: | | |
| github.event_name == 'pull_request' || | |
| (github.event_name == 'issue_comment' && github.event.issue.pull_request) | |
| uses: actions-ecosystem/action-create-comment@e23bc59fbff7aac7f9044bd66c2dc0fe1286f80b # v1.0.0 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| body: | | |
| ## :rocket: Performance test result | |
| **Test Suite**: ${{ steps.choose_perf.outputs.suites }} (${{ steps.choose_perf.outputs.tags }}) | |
| ${{ join(fromJSON(steps.charts.outputs.markdown_urls), ' ') }} | |
| <details><summary>Click to expand</summary> | |
| ``` | |
| ${{ steps.output.outputs.result }} | |
| Kong error logs are also available in Github Actions artifacts. | |
| ``` | |
| </details> | |
| [Download Artifacts](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts) for detailed results and interactive SVG flamegraphs. |