slack-workflow-queue.yml

name: Schedule Wait Queue Check

# Check GitHub's API if there are any workflows in wait queue for longer then 5 minutes

on:
  pull_request:
    paths:
      - '.github/workflows/slack-workflow-queue.yml'
  schedule:
    - cron: "*/20 * * * *"

jobs:
  check-queue:
    name: Check Workflow Wait Queue
    runs-on:
      labels: dind-small
    container:
      image: ghcr.io/dfinity/minimal-runner-image:0.1
    steps:

      - name: Check for workflows in wait queue
        id: check
        shell: bash
        run: |
          set -euo pipefail

          GITHUB_API_URL="https://api.github.com/repos/${{ github.repository }}/actions/runs"
          WAIT_THRESHOLD_MINUTES=20
          WAIT_THRESHOLD_SECONDS=$((WAIT_THRESHOLD_MINUTES * 60))

          # Fetch workflow runs
          WORKFLOW_RUNS=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" $GITHUB_API_URL)
          # Check for queued workflows
          CURRENT_TIME=$(date +%s)
          WORKFLOWS_IN_QUEUE=$(echo $WORKFLOW_RUNS | jq -r '.workflow_runs[] | select(.status == "queued") | {id: .id, name: .name, run_started_at: .run_started_at}')

          set -x
          MESSAGE=""
          for WORKFLOW in $(echo "${WORKFLOWS_IN_QUEUE}" | jq -r '.id'); do
            RUN_STARTED_AT=$(echo "${WORKFLOWS_IN_QUEUE}" | jq -r "select(.id == ${WORKFLOW}) | .run_started_at")
            RUN_STARTED_TIME=$(date -d "$RUN_STARTED_AT" +%s)
            TIME_IN_QUEUE=$((CURRENT_TIME - RUN_STARTED_TIME))

            # exclude workflows with long wait times
            WORKFLOW_NAME=$(echo "${WORKFLOWS_IN_QUEUE}" | jq -r "select(.id == ${WORKFLOW}) | .name") # Extract name
            if [ "$WORKFLOW_NAME" = "Schedule Rust Benchmarks" ]; then
              continue
            fi

            if [ $TIME_IN_QUEUE -ge $WAIT_THRESHOLD_SECONDS ]; then
              WORKFLOW_URL="https://github.com/${{ github.repository }}/actions/runs/${WORKFLOW}"
              MESSAGE="${MESSAGE} <${WORKFLOW_URL}|${WORKFLOW}>"
            fi
          done

          if [ "$MESSAGE" !=  "" ]; then
            MESSAGE=":alert: <https://github.com/${{ github.repository }}/actions?query=is%3Aqueued|Workflows in Wait Queue> [> ${WAIT_THRESHOLD_MINUTES} min]: ${MESSAGE}"
          fi
          echo "message=${MESSAGE}" >> $GITHUB_OUTPUT

      - name: Post Slack Notification
        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
        if: steps.check.outputs.message != ''
        with:
          channel-id: eng-idx-alerts
          slack-message: ${{ steps.check.outputs.message }}
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_API_TOKEN }}