test262.yml

name: EcmaScript official test suite (test262)
on:
  push:
    branches:
      - main
    tags:
      - v*
  pull_request:
    branches:
      - main
  merge_group:
    types: [checks_requested]

jobs:
  run_test262:
    name: Run the test262 test suite
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout the repository
        uses: actions/checkout@v3
        with:
          submodules: true
          path: boa
      - name: Install the Rust toolchain
        uses: actions-rs/toolchain@v1.0.7
        with:
          toolchain: stable
          override: true
          profile: minimal
      - name: Cache cargo
        uses: actions/cache@v3
        with:
          path: |
            target
            ~/.cargo/git
            ~/.cargo/registry
          key: ${{ runner.os }}-cargo-test262-${{ hashFiles('**/Cargo.lock') }}

      # Run the test suite and upload the results
      - name: Checkout GitHub pages
        uses: actions/checkout@v3
        with:
          ref: gh-pages
          path: gh-pages

      - name: Run the test262 test suite
        run: |
          cd boa
          mkdir ../results
          cargo run --release --bin boa_tester -- run -v -o ../results/test262
          cd ..

      # Run the results comparison
      - name: Compare results
        if: github.event_name == 'pull_request'
        id: compare-non-vm
        shell: bash
        run: |
          cd boa
          comment="$(./target/release/boa_tester compare ../gh-pages/test262/refs/heads/main/latest.json ../results/test262/pull/latest.json -m)"
          echo "comment<<EOF" >> $GITHUB_OUTPUT
          echo "$comment" >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

      - name: Get the PR number
        if: github.event_name == 'pull_request'
        id: pr-number
        uses: kkak10/pr-number-action@v1.3

      - name: Find Previous Comment
        if: github.event_name == 'pull_request'
        uses: peter-evans/find-comment@v2
        id: previous-comment
        with:
          issue-number: ${{ steps.pr-number.outputs.pr }}
          body-includes: Test262 conformance changes

      - name: Update comment
        if: github.event_name == 'pull_request' && steps.previous-comment.outputs.comment-id
        uses: peter-evans/create-or-update-comment@v3
        continue-on-error: true
        with:
          comment-id: ${{ steps.previous-comment.outputs.comment-id }}
          body: |
            ### Test262 conformance changes

            ${{ steps.compare-non-vm.outputs.comment }}
            ${{ steps.compare-vm.outputs.comment }}
          edit-mode: replace

      - name: Write a new comment
        if: github.event_name == 'pull_request' && !steps.previous-comment.outputs.comment-id
        uses: peter-evans/create-or-update-comment@v3
        continue-on-error: true
        with:
          issue-number: ${{ steps.pr-number.outputs.pr }}
          body: |
            ### Test262 conformance changes

            ${{ steps.compare-non-vm.outputs.comment }}
            ${{ steps.compare-vm.outputs.comment }}

      # Commit changes to GitHub pages.
      - name: Commit files
        if: github.event_name == 'push'
        run: |
          cp -r ./results/test262/* ./gh-pages/test262/
          cd gh-pages
          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"
          git add test262
          git commit -m "Add new test262 results" -a
          cd ..
      - name: Upload results
        if: github.event_name == 'push'
        uses: ad-m/github-push-action@v0.6.0
        with:
          directory: gh-pages
          branch: gh-pages
          github_token: ${{ secrets.GITHUB_TOKEN }}