ecosystem-benchmark.yml

name: ecosystem-benchmark

on:
  workflow_dispatch:
    inputs:
      pr:
        type: number
        description: "Run Benchmark PR number"
        required: true
  push:
    branches:
      - main
    paths-ignore:
      - "**/*.md"
      - "website/**"
    tags-ignore:
      - "**"

jobs:
  get-runner-labels:
    name: Get Runner Labels
    uses: ./.github/workflows/get-runner-labels.yml

  build:
    name: Test Linux
    needs: [get-runner-labels]
    uses: ./.github/workflows/reusable-build.yml
    with:
      target: x86_64-unknown-linux-gnu
      native: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS == '"ubuntu-22.04"' }}
      runner: ${{ needs.get-runner-labels.outputs.LINUX_RUNNER_LABELS }}
      ref: ${{ github.event_name == 'workflow_dispatch' && format('pull/{0}/head', inputs.pr) || github.sha }}
      test: false
      bench: false

  create-comment:
    runs-on: ubuntu-latest
    outputs:
      comment-id: ${{ steps.create-comment.outputs.result }}
    steps:
      - id: create-comment
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          result-encoding: string
          script: |
            const url = `${context.serverUrl}//${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
            const urlLink = `[Open](${url})`
            const body = `⏳ Triggered benchmark: ${urlLink}`

            if (context.eventName === 'workflow_dispatch') {
              const { data: comment } = await github.rest.issues.createComment({
                issue_number: context.payload.inputs.pr,
                owner: context.repo.owner,
                repo: context.repo.repo,
                body
              })
              return comment.id
            }

            const { data: comment } = await github.rest.repos.createCommitComment({
              commit_sha: context.sha,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body
            })
            return comment.id

  bench:
    needs: [build]
    runs-on: [self-hosted, benchmark]
    outputs:
      diff-result: ${{ steps.run-benchmark.outputs.diff-result }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event_name == 'workflow_dispatch' && format('pull/{0}/head', inputs.pr) || github.sha }}

      - name: Clean
        uses: ./.github/actions/clean
        with:
          target: x86_64-unknown-linux-gnu

      - name: Download bindings
        uses: ./.github/actions/download-artifact
        with:
          name: bindings-x86_64-unknown-linux-gnu
          path: crates/node_binding/
          try-local-cache: false

      - name: Show restored binding
        shell: bash
        run: ls -lah crates/node_binding/*.node

      - name: Pnpm Cache
        uses: ./.github/actions/pnpm-cache

      - name: Build JS
        run: pnpm run build:js

      - name: Run rspack-ecosystem-benchmark
        id: run-benchmark
        run: |
          RSPACK_DIR=$(pwd)
          git clone --single-branch --depth 1 https://github.com/web-infra-dev/rspack-ecosystem-benchmark.git
          cd rspack-ecosystem-benchmark
          pnpm i
          RSPACK_DIR="$RSPACK_DIR" node bin/cli.js bench
          result=$(node bin/cli.js compare --base latest --current current)
          echo "$result"
          echo "diff-result=${result//$'\n'/'@@'}" >> $GITHUB_OUTPUT

  comment-compare-results:
    runs-on: ubuntu-latest
    needs: [create-comment, bench]
    if: ${{ !cancelled() }}
    steps:
      - uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const diffResult = `${{ needs.bench.outputs.diff-result }}`

            let result = "task ${{ needs.bench.result }}"
            if (diffResult) {
              result = diffResult.replace(/@@/g, "\n");
            }

            const url = `${context.serverUrl}//${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`
            const urlLink = `[Open](${url})`
            const body = `
            📝 Benchmark detail: ${urlLink}

            ${result}
            `

            if (context.eventName === 'workflow_dispatch') {
              await github.rest.issues.updateComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: `${{ needs.create-comment.outputs.comment-id }}`,
                body
              })
            } else {
              await github.rest.repos.updateCommitComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                comment_id: `${{ needs.create-comment.outputs.comment-id }}`,
                body,
              });
            }

            if (result.includes("Threshold exceeded")) {
              console.log("Some benchmark cases exceed the threshold, please visit the previous step for more information");
              process.exit(1);
            }