visual-comparison.yaml

name: Visual Comparison


on:
  pull_request:
    types: [opened, reopened, synchronize, labeled]
    branches-ignore: ["version-*", "release-*"]


env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  PALETTE_API_KEY: ${{ secrets.PALETTE_API_KEY }}
  FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }}
  ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
  ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
  ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
  ALGOLIA_INDEX_NAME: ${{ secrets.ALGOLIA_INDEX_NAME }}
  HTML_REPORT_URL_PATH: reports/${{ github.head_ref }}/${{ github.run_id }}/${{ github.run_attempt }}
  DISABLE_PACKS_INTEGRATIONS: ${{ secrets.DISABLE_PACKS_INTEGRATIONS }}
  DISABLE_SECURITY_INTEGRATIONS: ${{ secrets.DISABLE_SECURITY_INTEGRATIONS }} 
  DSO_AUTH_TOKEN: ${{ secrets.DSO_AUTH_TOKEN }}


concurrency:
  group: "visual-comparison-${{ github.workflow }}-${{ github.head_ref }}"
  cancel-in-progress: true


jobs:
  run-ci:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    if: ${{ !github.event.pull_request.draft && github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' && contains(github.event.pull_request.labels.*.name, 'visual-tests') }}
    steps:
      # If the condition above is not met, aka, the PR is not in draft status, then this step is skipped.
      # Because this step is part of the critical path, omission of this step will result in remaining CI steps not gettinge executed.
      # As of 8/8/2022 there is now way to enforce this beahvior in GitHub Actions CI.
      - run: |
          echo "GITHUB_BASE_REF: ${{ github.base_ref }}"
          echo "GITHUB_HEAD_REF: ${{ github.head_ref }}"
          echo "GITHUB_REF_NAME: ${{ github.ref_name }}"
          echo "HTML_REPORT_URL_PATH ${{ env.HTML_REPORT_URL_PATH }}"
          exit 0



  create-assets:
    name: create-assets
    needs: [run-ci]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Setup Node.js Environment
        uses: actions/setup-node@v4
        with:
          node-version: "20"
          cache: "npm"

      - name: Post PR Comment
        uses: mshick/add-pr-comment@v2
        with:
          message: |
              🤖 Starting the visual tests. This will take approximately an hour.
          refresh-message-position: true

      - name: Install Dependencies
        uses: Wandalen/wretry.action@v3
        with: 
            command: npm ci
            attempt_limit: 3
            attempt_delay: 60000 # 1 minute

      - name: Build
        run: |
          touch .env
          make build-ci
      
      - name: Build with cached packs
        if: ${{ env.BUILD_EXIT_CODE == '5' }}
        uses: ./.github/actions/build-cached-packs
        with:
          gh-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Upload Build
        uses: actions/upload-artifact@v4
        with:
          name: "build"
          path: |
            build/
          if-no-files-found: error
          retention-days: 1



  take-screenshots:
    name: Visual Comparison
    needs: [run-ci, create-assets]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        shardIndex: [1, 2, 3, 4]
        shardTotal: [4]
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4

      - name: Setup Node.js environment
        uses: actions/setup-node@v4
        with:
          node-version: "20"
          cache: "npm"

      - name: Install Dependencies
        uses: Wandalen/wretry.action@v3
        with: 
            command: npm ci
            attempt_limit: 3
            attempt_delay: 60000 # 1 minute

      - name: Install Playwright browsers
        run: npx playwright install --with-deps chromium

      - name: Download Build
        uses: Wandalen/wretry.action@v3
        with: 
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name: build
            path: build
          attempt_delay: 60000 # 1 minute

      - name: Download Reference Screenshots
        run:  ./scripts/screenshot_artifacts.sh ./screenshots

      - name: Take screenshots with Playwright
        run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} || exit 0

      - name: Upload blob report to GitHub Actions Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: blob-report-${{ matrix.shardIndex }}
          path: blob-report
          retention-days: 1


  merge-reports:
    name: Merge Reports
    needs: [take-screenshots]
    runs-on: ubuntu-latest
    steps:
    - name: Check out repository code
      uses: actions/checkout@v4

    - name: Setup Node.js environment
      uses: actions/setup-node@v4
      with:
        node-version: "20"
        cache: "npm"

    - name: Install Dependencies
      uses: Wandalen/wretry.action@v3
      with: 
          command: npm ci
          attempt_limit: 3
          attempt_delay: 60000 # 1 minute
    
    - name: Download blob reports from GitHub Actions Artifacts
      uses: actions/download-artifact@v4
      with:
        path: all-blob-reports
        pattern: blob-report-*
        merge-multiple: true

    - name: Merge into HTML Report
      run: npx playwright merge-reports --reporter html ./all-blob-reports

    - name: Upload Report
      uses: actions/upload-artifact@v4
      id: report
      with:
        name: "report"
        path: |
          playwright-report/
        if-no-files-found: error
        retention-days: 1  


  publish_report:
    name: Publish HTML Report
    needs: [run-ci, take-screenshots, merge-reports]
    runs-on: ubuntu-latest
    continue-on-error: true    
    steps: 

    - name: Checkout GitHub Pages Branch
      uses: actions/checkout@v4
      with:
        ref: gh-pages  

    - name: Set Git User
          # see: https://github.com/actions/checkout/issues/13#issuecomment-724415212
      run: |
        git config --global user.name "github-actions[bot]"
        git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"

    - name: Download zipped HTML report
      uses: actions/download-artifact@v4
      with:
        name: report
        path: ${{ env.HTML_REPORT_URL_PATH }}

    - name: Push HTML Report
      timeout-minutes: 5
      run: |
        git add .
        git commit -m "workflow: add HTML report for run-id ${{ github.run_id }} (attempt:  ${{ github.run_attempt }})"
        while true; do
          git pull --rebase
          if [ $? -ne 0 ]; then
            echo "Failed to rebase. Please review manually."
            exit 1
          fi

          git push
          if [ $? -eq 0 ]; then
            echo "Successfully pushed HTML report to repo."
            exit 0
          fi
        done


    - name: DNS Wait
      run:  sleep 120

        
    - name: GH Pages URL
      run: |
        echo "::notice title= Published Playwright Test Report::https://spectrocloud.github.io/librarium/$HTML_REPORT_URL_PATH"


    - name: Post Report URL Comment
      uses: mshick/add-pr-comment@v2
      with:
        message: |
            📋 Visual Report for branch ${{ github.head_ref }} with CI run ${{ github.run_id }} and attempt ${{ github.run_attempt }} is ready at
              https://spectrocloud.github.io/librarium/${{env.HTML_REPORT_URL_PATH}}

              💡 You may have to wait for DNS to resolve or the GitHub Pages job to complete. You can view the progress of the GitHub Pages job [here](https://github.com/spectrocloud/librarium/actions/workflows/pages/pages-build-deployment).
        message-failure: |
            👎 Uh oh! Unable to publish Visual Report URL.
        refresh-message-position: true
        update-only: true