perf.yml

name: Performance Test

on:
  pull_request:
  schedule:
  # don't know the timezone but it's daily at least
  - cron:  '0 7 * * *'

env:
  terraform_version: '1.2.4'
  HAS_ACCESS_TO_GITHUB_TOKEN: ${{ github.event_name != 'pull_request' || (github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') }}
  BUILD_ROOT: ${{ github.workspace }}/bazel-bin/build

  # only for pr
  GHA_CACHE: ${{ github.event_name == 'pull_request' }}

jobs:
  build-packages:
    name: Build dependencies
    runs-on: ubuntu-22.04
    if: |
      github.event_name == 'schedule' ||
      (github.event_name == 'pull_request' && startsWith(github.event.pull_request.title, 'perf(')) ||
      (github.event_name == 'issue_comment' && github.event.action == 'created' &&
        github.event.issue.pull_request &&
        contains('["OWNER", "COLLABORATOR", "MEMBER"]', github.event.comment.author_association) &&
        (startsWith(github.event.comment.body, '/perf') || startsWith(github.event.comment.body, '/flamegraph'))
      )

    outputs:
      cache-key: ${{ steps.cache-key.outputs.cache-key }}

    steps:
    - name: Checkout Kong source code
      uses: actions/checkout@v4

    - name: Generate cache key
      id: cache-key
      uses: ./.github/actions/build-cache-key
      with:
        prefix: perf

    - name: Lookup build cache
      id: cache-deps
      uses: actions/cache@v4
      with:
        path: ${{ env.BUILD_ROOT }}
        key: ${{ steps.cache-key.outputs.cache-key }}

    - name: Install packages
      if: steps.cache-deps.outputs.cache-hit != 'true'
      run: sudo apt update && sudo apt install libyaml-dev valgrind libprotobuf-dev

    - name: Build Kong
      if: steps.cache-deps.outputs.cache-hit != 'true'
      env:
        GH_TOKEN: ${{ github.token }}
      run: |
        make build-kong
        BUILD_PREFIX=$BUILD_ROOT/kong-dev
        export PATH="$BUILD_PREFIX/bin:$BUILD_PREFIX/openresty/nginx/sbin:$BUILD_PREFIX/openresty/bin:$PATH"
        chmod +rw -R $BUILD_PREFIX
        nginx -V
        ldd $(which nginx)
        luarocks

    - name: Bazel Outputs
      uses: actions/upload-artifact@v3
      if: failure()
      with:
        name: bazel-outputs
        path: |
          bazel-out/_tmp/actions
        retention-days: 3

    - name: Build Dev Kong dependencies
      if: steps.cache-deps.outputs.cache-hit != 'true'
      run: |
        make install-dev-rocks

  perf:
    name: RPS, latency and flamegraphs
    runs-on: ubuntu-22.04
    needs: build-packages

    permissions:
      # required to send comment of graphs and results in the PR
      pull-requests: write

    if: |
      github.event_name == 'schedule' ||
      (github.event_name == 'pull_request' && startsWith(github.event.pull_request.title, 'perf(')) ||
      (github.event_name == 'issue_comment' && github.event.action == 'created' &&
        github.event.issue.pull_request &&
        contains('["OWNER", "COLLABORATOR", "MEMBER"]', github.event.comment.author_association) &&
        (startsWith(github.event.comment.body, '/perf') || startsWith(github.event.comment.body, '/flamegraph'))
      )

    # perf test can only run one at a time per repo for now
    concurrency:
      group: perf-ce

    steps:
    # set up mutex across CE and EE to avoid resource race
    - name: Set up mutex
      uses: ben-z/gh-action-mutex@9709ba4d8596ad4f9f8bbe8e0f626ae249b1b3ac # v1.0-alpha-6
      with:
        repository: "Kong/kong-perf-mutex-lock"
        branch: "gh-mutex"
        repo-token: ${{ secrets.PAT }}

    - name: Checkout Kong source code
      uses: actions/checkout@v4
      with:
        # Fetch all history for all tags and branches
        fetch-depth: 0

    - name: Load Cached Packages
      id: cache-deps
      if: env.GHA_CACHE == 'true'
      uses: actions/cache@v4
      with:
        path: ${{ env.BUILD_ROOT }}
        key: ${{ needs.build-packages.outputs.cache-key }}

    - name: Install performance test Dependencies
      run: |
        # in Kong repository
        sudo apt update && sudo apt install inkscape -y

        # terraform!
        wget https://releases.hashicorp.com/terraform/${{ env.terraform_version }}/terraform_${{ env.terraform_version }}_linux_amd64.zip
        unzip terraform_${{ env.terraform_version }}_linux_amd64.zip
        sudo mv terraform /usr/bin/

    - name: Choose perf suites
      id: choose_perf
      run: |
        suites="$(printf '%s' "${{ github.event.comment.body }}" | awk '{print $1}')"
        tags="$(printf '%s' "${{ github.event.comment.body }}" | awk '{print $2}')"

        if [[ $suite == "/flamegraph" ]]; then
          suites="02-flamegraph"
          if [[ -z $tags ]]; then
            tags="simple"
          fi
        elif [[ $suite == "/perf" ]]; then
          suites="01-rps"
          if [[ -z $tags ]]; then
            tags="baseline,single_route"
          fi
        else
          # if not specified by comment, run both
          suites="01-rps 02-flamegraph"
          if [[ -z $tags ]]; then
            tags="baseline,single_route,simple"
          fi
        fi

        echo "suites=$suites" >> $GITHUB_OUTPUT
        echo "tags=$tags" >> $GITHUB_OUTPUT

    - uses: xt0rted/pull-request-comment-branch@d97294d304604fa98a2600a6e2f916a84b596dc7 # v1.4.1
      id: comment-branch
      if: github.event_name == 'issue_comment' && github.event.action == 'created'

    - name: Find compared versions
      id: compare_versions
      run: |
        pr_ref=$(echo "${{ github.event.pull_request.base.ref }}")
        custom_vers="$(printf '%s' "${{ github.event.comment.body }}" | awk '{print $3}')"

        if [[ ! -z "${pr_ref}" ]]; then
          vers="git:${{ github.head_ref }},git:${pr_ref}"
        elif [[ ! -z "${custom_vers}" ]]; then
          vers="${custom_vers}"
        elif [[ ! -z "${{ github.event.comment.body }}" ]]; then
          vers="git:${{ steps.comment-branch.outputs.head_ref}},git:${{ steps.comment-branch.outputs.base_ref}}"
        else # is cron job/on master
          vers="git:master,git:origin/master~10,git:origin/master~50"
        fi

        echo $vers

        echo "vers=$vers" >> $GITHUB_OUTPUT


    - name: Run Tests
      env:
        PERF_TEST_VERSIONS: ${{ steps.compare_versions.outputs.vers }}
        PERF_TEST_DRIVER: terraform
        PERF_TEST_TERRAFORM_PROVIDER: bring-your-own
        PERF_TEST_BYO_KONG_IP: ${{ secrets.PERF_TEST_BYO_KONG_IP }}
        PERF_TEST_BYO_WORKER_IP: ${{ secrets.PERF_TEST_BYO_WORKER_IP }}
        PERF_TEST_BYO_SSH_USER: gha
        PERF_TEST_USE_DAILY_IMAGE: true
        PERF_TEST_DISABLE_EXEC_OUTPUT: true
      timeout-minutes: 180
      run: |
        export PERF_TEST_BYO_SSH_KEY_PATH=$(pwd)/ssh_key
        echo "${{ secrets.PERF_TEST_BYO_SSH_KEY }}" > ${PERF_TEST_BYO_SSH_KEY_PATH}

        chmod 600 ${PERF_TEST_BYO_SSH_KEY_PATH}
        # setup tunnel for psql and admin port
        ssh -o StrictHostKeyChecking=no -o TCPKeepAlive=yes -o ServerAliveInterval=10 \
            -o ExitOnForwardFailure=yes -o ConnectTimeout=5 \
            -L 15432:localhost:5432 -L 39001:localhost:39001 \
            -i ${PERF_TEST_BYO_SSH_KEY_PATH} \
            ${PERF_TEST_BYO_SSH_USER}@${PERF_TEST_BYO_KONG_IP} tail -f /dev/null &
        sleep 5

        sudo iptables -t nat -I OUTPUT -p tcp --dport 5432  -d ${PERF_TEST_BYO_KONG_IP} -j DNAT --to 127.0.0.1:15432
        sudo iptables -t nat -I OUTPUT -p tcp --dport 39001 -d ${PERF_TEST_BYO_KONG_IP} -j DNAT --to 127.0.0.1:39001

        make dev # required to install other dependencies like bin/grpcurl
        source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
        for suite in ${{ steps.choose_perf.outputs.suites }}; do
          # Run each test individually, ngx.pipe doesn't like to be imported twice
          # maybe bin/busted --no-auto-insulate
          for f in $(find "spec/04-perf/$suite/" -type f); do
            bin/busted "$f" \
              -t "${{ steps.choose_perf.outputs.tags }}"
          done
        done

    - name: Teardown
      # Note: by default each job has if: ${{ success() }}
      if: always()
      env:
        PERF_TEST_VERSIONS: git:${{ github.sha }}
        PERF_TEST_DRIVER: terraform
        PERF_TEST_TERRAFORM_PROVIDER: bring-your-own
        PERF_TEST_BYO_KONG_IP: ${{ secrets.PERF_TEST_BYO_KONG_IP }}
        PERF_TEST_BYO_WORKER_IP: ${{ secrets.PERF_TEST_BYO_WORKER_IP }}
        PERF_TEST_BYO_SSH_USER: gha
        PERF_TEST_TEARDOWN_ALL: true
      run: |
        export PERF_TEST_BYO_SSH_KEY_PATH=$(pwd)/ssh_key
        echo "${{ secrets.PERF_TEST_BYO_SSH_KEY }}" > ${PERF_TEST_BYO_SSH_KEY_PATH}

        make dev # required to install other dependencies like bin/grpcurl
        source ${{ env.BUILD_ROOT }}/kong-dev-venv.sh
        bin/busted spec/04-perf/99-teardown/

        rm -f ${PERF_TEST_BYO_SSH_KEY_PATH}

    - name: Generate high DPI graphs
      if: always()
      run: |
        for i in $(ls output/*.svg); do
          inkscape --export-area-drawing --export-png="${i%.*}.png" --export-dpi=300 -b FFFFFF $i
        done

    - uses: actions/setup-python@v5
      with:
        python-version: '3.10'
        cache: 'pip'

    - name: Generate plots
      if: always()
      run: |
        cwd=$(pwd)
        cd spec/helpers/perf/charts/
        pip install -r requirements.txt
        for i in $(ls ${cwd}/output/*.data.json); do
          python ./charts.py $i -o "${cwd}/output/"
        done

    - name: Save results
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: perf-results
        path: |
          output/
          !output/**/*.log

        retention-days: 31

    - name: Save error logs
      uses: actions/upload-artifact@v3
      if: always()
      with:
        name: error_logs
        path: |
          output/**/*.log
        retention-days: 31

    - name: Output
      if: always()
      id: output
      run: |
        if [[ "${{ steps.choose_perf.outputs.suites }}" =~ "02-flamegraph" ]]; then
          result="Please see Github Actions artifacts for flamegraphs.

          "
        fi

        result="${result}$(cat output/result.txt)" || true

        # https://github.community/t/set-output-truncates-multiline-strings/16852/2
        result="${result//'%'/'%25'}"
        result="${result//$'\n'/'%0A'}"
        result="${result//$'\r'/'%0D'}"

        echo "result=$results" >> $GITHUB_OUTPUT

    - name: Upload charts
      if: always()
      id: charts
      uses: devicons/public-upload-to-imgur@352cf5f2805c692539a96cfe49a09669e6fca88e # v2.2.2
      continue-on-error: true
      with:
        path: output/*.png
        client_id: ${{ secrets.PERF_TEST_IMGUR_CLIENT_ID }}

    - name: Comment
      if: |
        github.event_name == 'pull_request' ||
        (github.event_name == 'issue_comment' && github.event.issue.pull_request)
      uses: actions-ecosystem/action-create-comment@e23bc59fbff7aac7f9044bd66c2dc0fe1286f80b # v1.0.0
      with:
        github_token: ${{ secrets.GITHUB_TOKEN }}
        body: |
          ## :rocket: Performance test result

          **Test Suite**: ${{ steps.choose_perf.outputs.suites }} (${{ steps.choose_perf.outputs.tags }})

          ${{ join(fromJSON(steps.charts.outputs.markdown_urls), '     ') }}

          <details><summary>Click to expand</summary>

          ```
          ${{ steps.output.outputs.result }}

          Kong error logs are also available in Github Actions artifacts.
          ```

          </details>

          [Download Artifacts](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts) for detailed results and interactive SVG flamegraphs.