bench-turborepo.yml

name: Benchmark Turborepo

on:
  workflow_dispatch:
  workflow_run:
    # Make sure this matches the name of the workflow in ./github/workflows/turborepo-release.yml.
    workflows: [Turborepo Release]
    types:
      - completed
  push:
    branches:
      - main
    paths:
      - "cli/**"
      - crates/turborepo*/**
      # - "benchmark/**" (we don't need to run benchmarks when the benchmark changes, next push will capture it)

jobs:
  # benchmark:
  #   name: Run Benchmarks
  #   timeout-minutes: 60
  #   runs-on: ubuntu-latest

  #   steps:
  #     - uses: actions/checkout@v3
  #     - uses: ./.github/actions/setup-node
  #     - uses: ./.github/actions/setup-go
  #       with:
  #         github-token: "${{ secrets.GITHUB_TOKEN }}"

  #     - uses: ./.github/actions/setup-turborepo-environment
  #       with:
  #         target: ubuntu-latest
  #         github-token: "${{ secrets.GITHUB_TOKEN }}"

  #     - name: Build
  #       run: cd cli && make turbo-prod

  #     - name: Run benchmarks
  #       run: pnpm -F benchmark benchmark

  #     - name: Save to Tinybird
  #       run: |
  #         curl \
  #           -i \
  #           -F "ndjson=@./benchmark/tinybird.ndjson" \
  #           -X POST \
  #           -H 'Authorization: Bearer ${{ secrets.TINYBIRD_TOKEN }}' \
  #           'https://api.us-east.tinybird.co/v0/datasources?format=ndjson&name=turbo_benchmarks&mode=append'

  time-to-first-task:
    name: Bench TTFT
    timeout-minutes: 60
    runs-on: ${{ matrix.os.runner }}
    strategy:
      fail-fast: false
      matrix:
        os:
          - name: ubuntu
            runner: ubuntu-latest
          - name: macos
            runner: macos-latest
          - name: windows
            runner: windows-latest

    steps:
      - uses: actions/checkout@v3
      - name: Set filename for profile
        id: filename
        shell: bash
        run: |
          echo 'filename=${{ matrix.os.runner }}.json' >> "$GITHUB_OUTPUT"

      - name: Debug filename
        shell: bash
        run: echo "${{ steps.filename.outputs.filename }}"

      # setup Rust on windows, because apparently setup-turborepo-environment doesn't do this
      # TODO: put this logic in ./github/actions/setup-rust and use that instead
      - name: Add rustup for windows
        if: ${{ matrix.os.runner == 'windows-latest'}}
        run: rustup target add x86_64-pc-windows-gnu

      - uses: ./.github/actions/setup-turborepo-environment
        with:
          target: ${{ matrix.os.runner }}
          github-token: "${{ secrets.GITHUB_TOKEN }}"

      - name: Build Turborepo from source
        run: cd cli && make turbo-prod

      - name: Run benchmarks
        shell: bash
        # ttft script will normalize filepath and place the profile in the benchmark directory.
        run: pnpm -F benchmark ttft "${{ steps.filename.outputs.filename }}"

      - name: Upload Artifacts
        uses: actions/upload-artifact@v3
        with:
          name: profiles # This name will be the folder each file gets downloaded to
          path: "benchmark/${{ steps.filename.outputs.filename }}" # cwd is root of the repository, so we need the benchmark/ prefixed path
          if-no-files-found: error

  # Send each of the profiles generated from the previous job to TinyBird
  # We'll wait for all profiles to complete before sending.
  send-to-tinybird:
    name: Send to Tinybird
    needs: [time-to-first-task]
    runs-on: ubuntu-latest
    env:
      TINYBIRD_TOKEN: ${{secrets.TINYBIRD_TOKEN}}

    steps:
      - uses: actions/checkout@v3

      - name: "Setup Node"
        uses: ./.github/actions/setup-node

      - name: Download profiles
        uses: actions/download-artifact@v3
        with:
          path: benchmark

      - name: List Downloaded Profiles
        shell: bash
        run: ls benchmark/profiles

      - name: Adjust data
        shell: bash
        run: |
          jq -f benchmark/src/fold.jq < benchmark/profiles/ubuntu-latest.json > benchmark/profiles/ubuntu-ttft.json
          jq -f benchmark/src/fold.jq < benchmark/profiles/macos-latest.json > benchmark/profiles/macos-ttft.json
          jq -f benchmark/src/fold.jq < benchmark/profiles/windows-latest.json > benchmark/profiles/windows-ttft.json

      - name: Debug TTFT Data
        shell: bash
        run: |
          cat benchmark/profiles/ubuntu-ttft.json
          cat benchmark/profiles/macos-ttft.json
          cat benchmark/profiles/windows-ttft.json

      - name: Send data
        shell: bash
        run: |
          node -r esbuild-register ./benchmark/src/ttft-send.ts benchmark/profiles/ubuntu-ttft.json ${{github.run_id}}
          node -r esbuild-register ./benchmark/src/ttft-send.ts benchmark/profiles/macos-ttft.json ${{github.run_id}}
          node -r esbuild-register ./benchmark/src/ttft-send.ts benchmark/profiles/windows-ttft.json ${{github.run_id}}

  # Post results to slack
  post-to-slack:
    name: Post to Slack
    needs: [time-to-first-task]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: "Setup Node"
        uses: ./.github/actions/setup-node

      - name: Download profiles
        uses: actions/download-artifact@v3
        with:
          path: benchmark

      - name: List Downloaded Profiles
        shell: bash
        run: ls benchmark/profiles

      - name: Adjust data
        shell: bash
        run: |
          jq -f benchmark/src/fold.jq < benchmark/profiles/ubuntu-latest.json > benchmark/profiles/ubuntu-ttft.json
          jq -f benchmark/src/fold.jq < benchmark/profiles/macos-latest.json > benchmark/profiles/macos-ttft.json
          jq -f benchmark/src/fold.jq < benchmark/profiles/windows-latest.json > benchmark/profiles/windows-ttft.json

      - name: Debug TTFT Data
        shell: bash
        run: |
          cat benchmark/profiles/ubuntu-ttft.json
          cat benchmark/profiles/macos-ttft.json
          cat benchmark/profiles/windows-ttft.json

      # TODO: compare results to previous data and only post regressions
      - name: Create Slack Payload
        shell: bash
        run: node -r esbuild-register ./benchmark/src/ttft-slack.ts ${{github.run_id}}

      - name: Debug Slack Payload
        shell: bash
        run: cat benchmark/slack-payload.json

      - name: Post To Slack
        run: echo "Done"
        # uses: slackapi/slack-github-action@v1.23.0
        # with:
        #   payload-file-path: "benchmark/slack-payload.json"
        env:
          SLACK_WEBHOOK_URL: "${{ secrets.TURBOREPO_PERF_BENCHMARK_SLACK_WEBHOOK_URL }}"