Added support for skipping tests using semver range.

Signed-off-by: dblock <[email protected]>

.github/workflows/test-spec.yml

@@ -21,14 +21,18 @@ jobs:
     strategy:
       matrix:
         entry:
-          - {version: 2.15.0, hub: 'opensearchproject'}
+          - {version: 2.0.0, admin_password: 'admin'}
+          - {version: 2.15.0}
           - {version: 2.16.0, hub: 'opensearchstaging'}
-    name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub }})
+
+    name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }})
     runs-on: ubuntu-latest
+
     env:
-      OPENSEARCH_DOCKER_HUB_PROJECT: ${{ matrix.entry.hub }}
+      OPENSEARCH_DOCKER_HUB_PROJECT: ${{ matrix.entry.hub || 'opensearchproject' }}
       OPENSEARCH_VERSION: ${{ matrix.entry.version }}
-      OPENSEARCH_PASSWORD: myStrongPassword123!
+      OPENSEARCH_PASSWORD: ${{ matrix.entry.admin_password || 'myStrongPassword123!' }}
+
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4

SPECIFICATION_TESTING.md

@@ -1,9 +1,10 @@
 <!-- TOC -->
-* [Specification Testing](#specification-testing)
-  * [Running Spec Tests Locally](#running-spec-tests-locally)
-  * [Writing Spec Tests](#writing-spec-tests)
-    * [Simple Test Story](#simple-test-story)
-    * [Using Output from Previous Chapters](#using-output-from-previous-chapters)
+- [Specification Testing](#specification-testing)
+  - [Running Spec Tests Locally](#running-spec-tests-locally)
+  - [Writing Spec Tests](#writing-spec-tests)
+    - [Simple Test Story](#simple-test-story)
+    - [Using Output from Previous Chapters](#using-output-from-previous-chapters)
+    - [Managing Versions](#managing-versions)
 <!-- TOC -->
 
 # Specification Testing
@@ -133,4 +134,22 @@ Consider the following chapters in [ml/model_groups](tests/ml/model_groups.yaml)
     response:
       status: 200
 ```
-As you can see, the `output` field in the first chapter saves the `model_group_id` from the response body. This value is then used in the subsequent chapters to query and delete the model group.
+As you can see, the `output` field in the first chapter saves the `model_group_id` from the response body. This value is then used in the subsequent chapters to query and delete the model group.
+
+### Managing Versions
+
+It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver range in your tests as follows.
+
+```yaml
+- synopsis: Search with `phase_took` added in OpenSearch 2.12.
+  version: '>= 2.12.0'
+  path: /{index}/_search
+  parameters:
+    index: movies
+    cancel_after_time_interval: 10s
+  method: POST
+  response:
+    status: 200
+```
+
+The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests.

json_schemas/test_story.schema.yaml

@@ -76,6 +76,8 @@ definitions:
         $ref: '#/definitions/RequestBody'
       output:
         $ref: '#/definitions/Output'
+      version:
+        $ref: '#/definitions/Version'
     required: [path, method]
 
   Output:
@@ -90,6 +92,11 @@ definitions:
     additionalProperties:
       type: string
 
+  Version:
+    description: |
+      The semver range to execute the chapter against.
+    type: string
+
   RequestBody:
     type: object
     properties:

spec/schemas/_core.search.yaml

@@ -852,6 +852,7 @@ components:
         _shards:
           $ref: '_common.yaml#/components/schemas/ShardStatistics'
         phase_took:
+          x-version-added: '2.12.0'
           $ref: '_common.yaml#/components/schemas/PhaseTook'
         hits:
           $ref: '#/components/schemas/HitsMetadata'

tests/_core/search.yaml

@@ -87,11 +87,19 @@ chapters:
                 director: Bennett Miller
                 title: Moneyball
                 year: 2011
-  - synopsis: Search with parameters.
+  - synopsis: Search with cancel_after_time_interval.
     path: /{index}/_search
     parameters:
       index: movies
       cancel_after_time_interval: 10s
+    method: POST
+    response:
+      status: 200
+  - synopsis: Search with phase_took.
+    version: '>= 2.12.0'
+    path: /{index}/_search
+    parameters:
+      index: movies
       phase_took: true
     method: POST
     response:

tools/src/tester/StoryEvaluator.ts

@@ -14,6 +14,7 @@ import { overall_result } from './helpers'
 import { StoryOutputs } from './StoryOutputs'
 import SupplementalChapterEvaluator from './SupplementalChapterEvaluator'
 import { ChapterOutput } from './ChapterOutput'
+import * as semver from 'semver'
 
 export default class StoryEvaluator {
   private readonly _chapter_evaluator: ChapterEvaluator
@@ -24,14 +25,14 @@ export default class StoryEvaluator {
     this._supplemental_chapter_evaluator = supplemental_chapter_evaluator
   }
 
-  async evaluate({ story, display_path, full_path }: StoryFile, dry_run: boolean = false): Promise<StoryEvaluation> {
+  async evaluate({ story, display_path, full_path }: StoryFile, version: string, dry_run: boolean = false): Promise<StoryEvaluation> {
     const variables_error = StoryEvaluator.check_story_variables(story, display_path, full_path)
     if (variables_error !== undefined) {
       return variables_error
     }
     const story_outputs = new StoryOutputs()
     const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? [], dry_run, story_outputs)
-    const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run, story_outputs)
+    const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, version, dry_run, story_outputs)
     const { evaluations: epilogues } = await this.#evaluate_supplemental_chapters(story.epilogues ?? [], dry_run, story_outputs)
     return {
       display_path,
@@ -44,12 +45,15 @@ export default class StoryEvaluator {
     }
   }
 
-  async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs): Promise<ChapterEvaluation[]> {
+  async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, version: string, dry_run: boolean, story_outputs: StoryOutputs): Promise<ChapterEvaluation[]> {
     const evaluations: ChapterEvaluation[] = []
     for (const chapter of chapters) {
       if (dry_run) {
         const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } })
+      } else if (chapter.version !== undefined && !semver.satisfies(version, chapter.version)) {
+        const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
+        evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because ${version} does not satisfy ${chapter.version}.`, error: undefined } })
       } else {
         const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs)
         has_errors = has_errors || evaluation.overall.result === Result.ERROR

tools/src/tester/TestRunner.ts

@@ -36,14 +36,16 @@ export default class TestRunner {
     let failed = false
     const story_files = this.#sort_story_files(this.#collect_story_files(resolve(story_path), '', ''))
     const evaluations: StoryEvaluation[] = []
+    let version = '2.99.0' // TODO: should be the version targeted by the OpenAPI spec
 
     if (!dry_run) {
       const info = await this._http_client.wait_until_available()
       console.log(`OpenSearch ${ansi.green(info.version.number)}\n`)
+      version = info.version.number
     }
 
     for (const story_file of story_files) {
-      const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, dry_run)
+      const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, dry_run)
       evaluations.push(evaluation)
       this._result_logger.log(evaluation)
       if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true

tools/src/tester/helpers.ts

@@ -10,8 +10,9 @@
 import { type Evaluation, Result } from './types/eval.types'
 
 export function overall_result(evaluations: Evaluation[]): Result {
+  if (evaluations.length == 0) return Result.PASSED
   if (evaluations.some(e => e.result === Result.ERROR)) return Result.ERROR
   if (evaluations.some(e => e.result === Result.FAILED)) return Result.FAILED
-  if (evaluations.some(e => e.result === Result.SKIPPED)) return Result.SKIPPED
+  if (evaluations.every(e => e.result === Result.SKIPPED)) return Result.SKIPPED
   return Result.PASSED
 }

tools/src/tester/types/story.types.ts

@@ -37,6 +37,14 @@ export type Parameter = (string | number | boolean)[] | string | number | boolea
  * via the `definition` "Payload".
  */
 export type Payload = {} | any[] | string | number | boolean;
+/**
+ * The minimum version to execute the chapter against.
+ *
+ *
+ * This interface was referenced by `Story`'s JSON-Schema
+ * via the `definition` "Version".
+ */
+export type Version = string;
 /**
  * This interface was referenced by `Story`'s JSON-Schema
  * via the `definition` "Chapter".
@@ -79,6 +87,7 @@ export interface ChapterRequest {
   };
   request_body?: RequestBody;
   output?: Output;
+  version?: Version;
 }
 /**
  * This interface was referenced by `Story`'s JSON-Schema

tools/tests/tester/fixtures/evals/semver.yaml

@@ -0,0 +1,46 @@
+display_path: semver.yaml
+full_path: tools/tests/tester/fixtures/stories/semver.yaml
+
+result: PASSED
+description: This story should pass.
+
+prologues: []
+epilogues: []
+
+chapters:
+  - title: This GET /_cat/health should run (default).
+    overall:
+      result: PASSED
+    request:
+      parameters:
+        format:
+          result: PASSED
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload_body:
+        result: PASSED
+      payload_schema:
+        result: PASSED
+  - title: This GET /_cat/health should run (~> 2.x).
+    overall:
+      result: PASSED
+    request:
+      parameters:
+        format:
+          result: PASSED
+      request_body:
+        result: PASSED
+    response:
+      status:
+        result: PASSED
+      payload_body:
+        result: PASSED
+      payload_schema:
+        result: PASSED
+  - title: This GET /_cat/health should be skipped (> 2.999.0).
+    overall:
+      result: SKIPPED
+      message: Skipped because 2.99.0 does not satisfy >= 2.999.0.

tools/tests/tester/fixtures/stories/semver.yaml

@@ -0,0 +1,21 @@
+$schema: ../../../../../json_schemas/test_story.schema.yaml
+
+description: This story should pass.
+chapters:
+  - synopsis: This GET /_cat/health should run (default).
+    method: GET
+    path: /_cat/health
+    parameters:
+      format: json
+  - synopsis: This GET /_cat/health should run (~> 2.x).
+    version: '~> 2.x'
+    method: GET
+    path: /_cat/health
+    parameters:
+      format: json
+  - synopsis: This GET /_cat/health should be skipped (> 2.999.0).
+    version: '>= 2.999.0'
+    method: GET
+    path: /_cat/health
+    parameters:
+      format: json

tools/tests/tester/helpers.test.ts

@@ -18,7 +18,8 @@ describe('helpers', () => {
   test('overall_result', () => {
     expect(overall_result(e(Result.PASSED, Result.SKIPPED, Result.FAILED, Result.ERROR))).toBe(Result.ERROR)
     expect(overall_result(e(Result.PASSED, Result.SKIPPED, Result.FAILED))).toBe(Result.FAILED)
-    expect(overall_result(e(Result.PASSED, Result.SKIPPED))).toBe(Result.SKIPPED)
+    expect(overall_result(e(Result.PASSED, Result.SKIPPED))).toBe(Result.PASSED)
+    expect(overall_result(e(Result.SKIPPED, Result.SKIPPED))).toBe(Result.SKIPPED)
     expect(overall_result(e(Result.PASSED))).toBe(Result.PASSED)
     expect(overall_result(e())).toBe(Result.PASSED)
   })

tools/tests/tester/helpers.ts

@@ -138,5 +138,5 @@ export async function load_actual_evaluation (evaluator: StoryEvaluator, name: s
     full_path,
     display_path: `${name}.yaml`,
     story: read_yaml(full_path)
-  }))
+  }, '2.99.0'))
 }

tools/tests/tester/integ/StoryEvaluator.test.ts

@@ -40,3 +40,9 @@ test('error/chapter_error', async () => {
   const expected = load_expected_evaluation('error/chapter_error')
   expect(actual).toEqual(expected)
 })
+
+test('semver', async () => {
+  const actual = await load_actual_evaluation(story_evaluator, 'semver')
+  const expected = load_expected_evaluation('semver')
+  expect(actual).toEqual(expected)
+})