eamxx-sa-testing.yml

name: eamxx-sa

on:
  # Runs on PRs against master
  pull_request:
    branches: [ master ]
    types: [opened, synchronize, ready_for_review, reopened]

  # Manual run is used to bless
  workflow_dispatch:
    inputs:
      job_to_run:
        description: 'Job to run'
        required: true
        type: choice
        options:
          - gcc-openmp
          - gcc-cuda
          - all
      bless:
        description: 'Generate baselines'
        required: true
        type: boolean
      submit:
        description: 'Force cdash submission'
        required: true
        type: boolean

  # Add schedule trigger for nightly runs at midnight MT (Standard Time)
  schedule:
    - cron: '0 7 * * *'  # Runs at 7 AM UTC, which is midnight MT during Standard Time

concurrency:
  # Two runs are in the same group if they are testing the same git ref
  #  - if trigger=pull_request, the ref is refs/pull/<PR_NUMBER>/merge
  #  - for other triggers, the ref is the branch tested
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  # Submit to cdash only for nightlies or if the user explicitly forced a submission via workflow dispatch
  submit: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.submit) }}

jobs:
  pre_process_pr:
    if: ${{ github.event_name == 'pull_request' }}
    runs-on: ubuntu-latest  # This job can run anywhere
    outputs:
      relevant_paths: ${{ steps.check_paths.outputs.value }}
      labels: ${{ steps.get_labels.outputs.labels }}
    steps:
      - name: Check files modified by PR
        id: check_paths
        run: |
          paths=(
            components/eamxx
            components/eam/src/physics/rrtmgp
            components/eam/src/physics/p3/scream
            components/eam/src/physics/cam
            components/eam/src/physics/rrtmgp/external
            externals/ekat
            externals/scorpio
            externals/haero
            externals/YAKL
            .github/workflows/eamxx-sa-testing.yml
          )
          pattern=$(IFS=\|; echo "${paths[*]}")

          # Use the GitHub API to get the list of changed files
          # There are page size limits, so do it in chunks
          page=1
          while true; do
            response=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
              "https://api.github.com/repos/E3SM-Project/scream/pulls/${{ github.event.number }}/files?per_page=100&page=$page")

            # Check if the response is empty, and break if it is
            [ -z "$response" ] && break

            changed_files+=$(echo "$response" | grep -o '"filename": *"[^"]*"' | sed 's/"filename": *//; s/"//g')$'\n'

            # Check if there are more pages, and quite if there aren't
            [[ $(echo "$response" | jq '. | length') -lt 100 ]] && break

            page=$((page + 1))
          done

          # Check for matches and echo the matching files (or "" if none)
          matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "")
          if [[ -n "$matching_files" ]]; then
            echo "Found relevant files: $matching_files"
            echo "value=true" >> $GITHUB_OUTPUT
          else
            echo "No relevant files touched by this PR."
            echo "value=false" >> $GITHUB_OUTPUT
          fi
      - name: Retrieve PR labels
        id: get_labels
        run: |
          labels="${{ join(github.event.pull_request.labels.*.name, ',') }}"
          echo "labels=${labels}" >> $GITHUB_OUTPUT
  gcc-openmp:
    needs: [pre_process_pr]
    if: |
      !failure() && !cancelled() &&
      (
        github.event_name == 'schedule' ||
        (
          github.event_name == 'pull_request' &&
          needs.pre_process_pr.outputs.relevant_paths=='true' &&
          !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') &&
          !contains(needs.pre_process_pr.outputs.labels,'CI: skip openmp') &&
          !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') &&
          !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all')
        ) || (
          github.event_name == 'workflow_dispatch' &&
          github.event.inputs.job_to_run == 'gcc-openmp' || 
          github.event.inputs.job_to_run == 'all'
        )
      )
    runs-on:  [self-hosted, ghci-snl-cpu, gcc]
    strategy:
      fail-fast: false
      matrix:
        build_type: [sp, dbg, fpe, opt]
    name: gcc-openmp / ${{ matrix.build_type }}
    steps:
      - name: Check out the repository
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          show-progress: false
          submodules: recursive
      - name: Show action trigger
        uses: ./.github/actions/show-workflow-trigger
      - name: Set test-all inputs based on event specs
        run: |
          echo "generate=false" >> $GITHUB_ENV
          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            if [ "${{ inputs.bless }}" == "true" ]; then
              echo "generate=true" >> $GITHUB_ENV
            fi
          fi
      - name: Run tests
        uses: ./.github/actions/test-all-scream
        with:
          build_type: ${{ matrix.build_type }}
          machine: ghci-snl-cpu
          generate: ${{ env.generate }}
          submit: ${{ env.submit }}
          cmake-configs: Kokkos_ENABLE_OPENMP=ON
  gcc-cuda:
    needs: [pre_process_pr]
    if: |
      !failure() && !cancelled() &&
      (
        github.event_name == 'schedule' ||
        (
          github.event_name == 'pull_request' &&
          needs.pre_process_pr.outputs.relevant_paths=='true' &&
          !contains(needs.pre_process_pr.outputs.labels,'CI: skip gcc') &&
          !contains(needs.pre_process_pr.outputs.labels,'CI: skip cuda') &&
          !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-sa') &&
          !contains(needs.pre_process_pr.outputs.labels,'CI: skip eamxx-all')
        ) || (
          github.event_name == 'workflow_dispatch' &&
          github.event.inputs.job_to_run == 'gcc-cuda' ||
          github.event.inputs.job_to_run == 'all'
        )
      )
    runs-on:  [self-hosted, ghci-snl-cuda, cuda, gcc]
    strategy:
      fail-fast: false
      matrix:
        build_type: [sp, dbg, opt]
    name: gcc-cuda / ${{ matrix.build_type }}
    steps:
      - name: Check out the repository
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          show-progress: false
          submodules: recursive
      - name: Show action trigger
        uses: ./.github/actions/show-workflow-trigger
      - name: Set test-all inputs based on event specs
        run: |
          echo "generate=false" >> $GITHUB_ENV
          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            if [ "${{ inputs.bless }}" == "true" ]; then
              echo "generate=true" >> $GITHUB_ENV
            fi
          fi
      - name: Get CUDA Arch
        run: |
          # Ensure nvidia-smi is available
          if ! command -v nvidia-smi &> /dev/null; then
              echo "nvidia-smi could not be found. Please ensure you have Nvidia drivers installed."
              exit 1
          fi

          # Get the GPU model from nvidia-smi, and set env for next step
          gpu_model=$(nvidia-smi --query-gpu=name --format=csv,noheader | head -n 1)
          case "$gpu_model" in
              *"H100"*)
                  echo "Hopper=ON" >> $GITHUB_ENV
                  echo "CUDA_ARCH=90" >> $GITHUB_ENV
                  ARCH=90
                  ;;
              *"A100"*)
                  echo "Ampere=ON" >> $GITHUB_ENV
                  echo "CUDA_ARCH=80" >> $GITHUB_ENV
                  ;;
              *"V100"*)
                  echo "Volta=ON" >> $GITHUB_ENV
                  echo "CUDA_ARCH=70" >> $GITHUB_ENV
                  ;;
              *)
                  echo "Unsupported GPU model: $gpu_model"
                  exit 1
                  ;;
          esac
      - name: Run tests
        uses: ./.github/actions/test-all-scream
        with:
          build_type: ${{ matrix.build_type }}
          machine: ghci-snl-cuda
          generate: ${{ env.generate }}
          submit: ${{ env.submit }}
          cmake-configs: Kokkos_ARCH_HOPPER90=${{ env.Hopper }};Kokkos_ARCH_AMPERE80=${{ env.Ampere }};Kokkos_ARCH_VOLTA70=${{ env.Volta }};CMAKE_CUDA_ARCHITECTURES=${{ env.CUDA_ARCH }}