test-spec.yml

name: Test Spec

on:
  push:
    paths:
      - .github/workflows/test-spec.yml
      - package*.json
      - spec/**
      - tests/**
      - tools/src/tester/**
      - tsconfig.json
  pull_request:
    paths:
      - .github/workflows/test-spec.yml
      - package*.json
      - spec/**
      - tests/**
      - tools/src/tester/**
      - tsconfig.json

jobs:
  test-opensearch-spec:
    strategy:
      fail-fast: false
      matrix:
        entry:
          - version: 1.3.17
            admin_password: admin
          - version: 2.0.0
            admin_password: admin
          - version: 2.18.0
          - version: 2.18.0
            tests: plugins/index_state_management
          - version: 2.18.0
            tests: plugins/ml
          - version: 2.18.0
            tests: routing
          - version: 2.18.0
            tests: snapshot
          - version: 2.18.0
            tests: remote_store
          - version: 2.18.0
            tests: dangling
            url: http://localhost:9200
          - version: 2.18.0
            tests: plugins/replication
            url: http://localhost:9200
          - version: 2.18.0
            tests: plugins/streaming
          - version: 2.18.0
            tests: plugins/notifications
          - version: 2.18.0
            tests: plugins/query_insights
          - version: 2.18.0
            tests: plugins/workload-management
          - version: 2.18.0
            tests: plugins/analysis
          - version: 2.18.0
            tests: plugins/security
            cert: tests/plugins/security/.kirk.pem
            key: tests/plugins/security/.kirk-key.pem
          - version: 2.19.0
            hub: opensearchstaging
            ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc'
          - version: 3.0.0
            hub: opensearchstaging
            ref: '@sha256:727643acdfebed77bfdb26362dbcff536b7ea02a0cc4ae2da2521729171333de'

    name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }})
    runs-on: ubuntu-latest

    env:
      OPENSEARCH_DOCKER_HUB_PROJECT: ${{ matrix.entry.hub || 'opensearchproject' }}
      OPENSEARCH_DOCKER_REF: ${{ matrix.entry.ref }}
      OPENSEARCH_VERSION: ${{ matrix.entry.version }}
      OPENSEARCH_PASSWORD: ${{ matrix.entry.admin_password || 'myStrongPassword123!' }}
      OPENSEARCH_JAVA_OPTS: ${{ matrix.entry.opts }} -Xms8g -Xmx8g

    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4

      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '20'

      - name: Install Dependencies
        run: npm ci

      - name: Run OpenSearch Cluster
        working-directory: tests/${{ matrix.entry.tests || 'default' }}
        run: docker compose up -d

      - name: Generate Test Files Hash
        id: tests
        run: echo "hash=${{ matrix.entry.version }}-${{ hashFiles(format('tests/{0}', matrix.entry.tests || 'default')) }}" >> $GITHUB_OUTPUT

      - name: Run Tests
        run: |
          npm run test:spec -- \
            --opensearch-insecure \
            --opensearch-version=${{ matrix.entry.version }} \
            --coverage coverage/test-spec-coverage-${{ steps.tests.outputs.hash }}.json \
            --opensearch-url=${{ matrix.entry.url || 'https://localhost:9200'}} \
            --opensearch-cert=${{ matrix.entry.cert }} \
            --opensearch-key=${{ matrix.entry.key }} \
            --tests=tests/${{ matrix.entry.tests || 'default' }}

      - name: Get Container Logs
        if: failure()
        run: |
          for container_id in $(docker ps -aqf "ancestor=opensearchproject/opensearch:${{ matrix.entry.version }}"); do \
            echo Dumping logs from $container_id ... && \
            docker logs $container_id \
          ; done

      - name: Upload Test Coverage Results
        uses: actions/upload-artifact@v4
        with:
          name: coverage-${{ matrix.entry.version }}-${{ steps.tests.outputs.hash }}
          path: coverage/test-spec-coverage-${{ steps.tests.outputs.hash }}.json

  merge-coverage:
    runs-on: ubuntu-latest
    needs: test-opensearch-spec
    steps:
      - uses: actions/checkout@v4

      - name: Download Spec Coverage Data
        uses: actions/download-artifact@v4
        with:
          path: coverage

      - name: Collect Test File Names
        run: |
          find tests/. -name "*.yaml" | xargs realpath | jq -Rn '{ files: [inputs | "\(.)"] }' > test-files.json

      - name: Display Unused Test Files
        run: |
          jq -r -sc '
              (map(.stories) | add | unique) as $stories |
              (map(.files) | add | unique) as $all |
              $all-$stories |
              .[]
            ' $(find ./ -name "test-spec-coverage-*.json") test-files.json > ./coverage/files.txt
          cat ./coverage/files.txt | sed -e 's/^/::error::/'
          test ! -s ./coverage/files.txt || { echo "::error::Unused test files detected."; exit 1; }

      - name: Collect and Display Missing Test Paths
        run: |
          jq -r -sc '
              (map(.operations) | add | unique) as $all |
              (map(.evaluated_operations) | add | unique) as $evaluated |
              ($all-$evaluated | map({path: .path, method: .method, evaluated: false})) as $unevaluated |
              ($evaluated | map({path: .path, method: .method, evaluated: true})) as $evaluated |
              $evaluated+$unevaluated |
              sort_by(.path) |
              map(if .evaluated then "- [x] \(.method) \(.path)" else "- [ ] \(.method) \(.path)" end) |
              .[]
            ' $(find ./ -name "test-spec-coverage-*.json")

      - name: Combine Test Coverage Data
        shell: bash -eo pipefail {0}
        run: |
          jq -sc '
            (map(.operations) | add | unique | length) as $total_operations_count |
            (map(.evaluated_operations) | add | unique | length) as $evaluated_operations_count |
            { 
              total_operations_count: $total_operations_count,
              evaluated_operations_count: $evaluated_operations_count,
              evaluated_paths_pct: $evaluated_operations_count | (10000 * . / $total_operations_count | round / 100)
            }
          ' $(find ./coverage -name "test-spec-coverage-*.json") > ./coverage/coverage.json
          cat ./coverage/coverage.json

      - name: Construct Comment Data Payload
        if: github.event_name == 'pull_request'
        shell: bash -eo pipefail {0}
        run: |
          jq \
            --arg pr_number ${PR_NUMBER} \
            --slurpfile test_coverage ./coverage/coverage.json \
            --null-input '
            {
              "pr_number": ($pr_number),
              "comment_identifier": "## Spec Test Coverage Analysis",
              "template_name": "pr-test-coverage-analysis",
              "template_data": {
                "test_coverage": ($test_coverage[0])
              }
            }
            ' | tee pr-comment.json
        env:
          PR_NUMBER: ${{ github.event.pull_request.number }}

      - name: Upload PR Comment Payload
        uses: actions/upload-artifact@v4
        if: github.event_name == 'pull_request'
        with:
          name: pr-comment
          path: pr-comment.json