Use strong types for parsed stories.

Signed-off-by: dblock <[email protected]>
opensearch-project · Dec 11, 2024 · 6327d3d · 6327d3d
1 parent 5024aec
commit 6327d3d
Show file tree

Hide file tree

Showing 13 changed files with 145 additions and 91 deletions.
diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts
@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import { type Chapter, type ActualResponse, type Payload } from './types/story.types'
+import { type ActualResponse, type Payload } from './types/story.types'
 import { type ChapterEvaluation, type Evaluation, Result, EvaluationWithOutput } from './types/eval.types'
 import { type ParsedOperation } from './types/spec.types'
 import { overall_result } from './helpers'
@@ -21,6 +21,7 @@ import _ from 'lodash'
 import { Logger } from 'Logger'
 import { sleep, to_json } from '../helpers'
 import { APPLICATION_JSON } from "./MimeTypes";
+import { ParsedChapter } from './types/parsed_story.types'
 
 export default class ChapterEvaluator {
   private readonly logger: Logger
@@ -35,11 +36,11 @@ export default class ChapterEvaluator {
     this.logger = logger
   }
 
-  async evaluate(chapter: Chapter, skip: boolean, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
+  async evaluate(chapter: ParsedChapter, skip: boolean, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
     if (skip) return { title: chapter.synopsis, overall: { result: Result.SKIPPED } }
 
     const operation = this._operation_locator.locate_operation(chapter)
-    if (operation == null) return { title: chapter.synopsis, overall: { result: Result.FAILED, message: `Operation "${chapter.method.toString().toUpperCase()} ${chapter.path}" not found in the spec.` } }
+    if (operation == null) return { title: chapter.synopsis, overall: { result: Result.FAILED, message: `Operation "${chapter.method.toUpperCase()} ${chapter.path}" not found in the spec.` } }
 
     var tries = chapter.retry && chapter.retry?.count > 0 ? chapter.retry.count + 1 : 1
     var retry = 0
@@ -61,7 +62,7 @@ export default class ChapterEvaluator {
     return result
   }
 
-  async #evaluate(chapter: Chapter, operation: ParsedOperation, story_outputs: StoryOutputs, retries?: number): Promise<ChapterEvaluation> {
+  async #evaluate(chapter: ParsedChapter, operation: ParsedOperation, story_outputs: StoryOutputs, retries?: number): Promise<ChapterEvaluation> {
     const response = await this._chapter_reader.read(chapter, story_outputs)
     const params = this.#evaluate_parameters(chapter, operation, story_outputs)
     const request = this.#evaluate_request(chapter, operation, story_outputs)
@@ -85,10 +86,10 @@ export default class ChapterEvaluator {
     var result: ChapterEvaluation = {
       title: chapter.synopsis,
       operation: {
-        method: chapter.method.toString(),
+        method: chapter.method,
         path: chapter.path
       },
-      path: `${chapter.method.toString()} ${chapter.path}`,
+      path: `${chapter.method} ${chapter.path}`,
       overall: { result: overall_result(evaluations) },
       request: { parameters: params, request },
       response: {
@@ -110,7 +111,7 @@ export default class ChapterEvaluator {
     return result
   }
 
-  #evaluate_parameters(chapter: Chapter, operation: ParsedOperation, story_outputs: StoryOutputs): Record<string, Evaluation> {
+  #evaluate_parameters(chapter: ParsedChapter, operation: ParsedOperation, story_outputs: StoryOutputs): Record<string, Evaluation> {
     const parameters: Record<string, any> = story_outputs.resolve_value(chapter.parameters) ?? {}
     return Object.fromEntries(Object.entries(parameters).map(([name, parameter]) => {
       const schema = operation.parameters[name]?.schema
@@ -120,7 +121,7 @@ export default class ChapterEvaluator {
     }))
   }
 
-  #evaluate_request(chapter: Chapter, operation: ParsedOperation, story_outputs: StoryOutputs): Evaluation {
+  #evaluate_request(chapter: ParsedChapter, operation: ParsedOperation, story_outputs: StoryOutputs): Evaluation {
     if (chapter.request?.payload === undefined) return { result: Result.PASSED }
     const content_type = chapter.request.content_type ?? APPLICATION_JSON
     const schema = operation.requestBody?.content[content_type]?.schema
@@ -129,7 +130,7 @@ export default class ChapterEvaluator {
     return this._schema_validator.validate(schema, payload)
   }
 
-  #evaluate_status(chapter: Chapter, response: ActualResponse): Evaluation {
+  #evaluate_status(chapter: ParsedChapter, response: ActualResponse): Evaluation {
     const expected_status = chapter.response?.status ?? 200
     if (response.status === expected_status && response.error === undefined) return { result: Result.PASSED }
 
@@ -166,7 +167,7 @@ export default class ChapterEvaluator {
     return messages.length > 0 ? { result: Result.FAILED, message: _.join(messages, ', ') } : { result: Result.PASSED }
   }
 
-  #evaluate_payload_schema(chapter: Chapter, response: ActualResponse, operation: ParsedOperation): Evaluation {
+  #evaluate_payload_schema(chapter: ParsedChapter, response: ActualResponse, operation: ParsedOperation): Evaluation {
     const content_type = chapter.response?.content_type ?? APPLICATION_JSON
 
     if (response.content_type !== content_type) {

diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts
@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import { type ChapterRequest, type ActualResponse, type Parameter } from './types/story.types'
+import { type ActualResponse, type Parameter } from './types/story.types'
 import { type OpenSearchHttpClient } from '../OpenSearchHttpClient'
 import { type StoryOutputs } from './StoryOutputs'
 import { Logger } from 'Logger'
@@ -18,6 +18,7 @@ import CBOR from 'cbor'
 import SMILE from 'smile-js'
 import { APPLICATION_CBOR, APPLICATION_JSON, APPLICATION_SMILE, APPLICATION_YAML, TEXT_PLAIN } from "./MimeTypes";
 import _ from 'lodash'
+import { ParsedChapterRequest } from './types/parsed_story.types'
 
 export default class ChapterReader {
   private readonly _client: OpenSearchHttpClient
@@ -28,7 +29,7 @@ export default class ChapterReader {
     this.logger = logger
   }
 
-  async read (chapter: ChapterRequest, story_outputs: StoryOutputs): Promise<ActualResponse> {
+  async read (chapter: ParsedChapterRequest, story_outputs: StoryOutputs): Promise<ActualResponse> {
     const response: Record<string, any> = {}
     const resolved_params = story_outputs.resolve_params(chapter.parameters ?? {})
     const [url_path, params] = this.#parse_url(chapter.path, resolved_params)
@@ -37,10 +38,10 @@ export default class ChapterReader {
       story_outputs.resolve_value(chapter.request.payload),
       content_type
     ) : undefined
-    this.logger.info(`=> ${chapter.method.toString()} ${url_path} (${to_json(params)}) [${content_type}] ${_.compact([to_json(headers), to_json(request_data)]).join(' | ')}`)
+    this.logger.info(`=> ${chapter.method} ${url_path} (${to_json(params)}) [${content_type}] ${_.compact([to_json(headers), to_json(request_data)]).join(' | ')}`)
     await this._client.request({
       url: url_path,
-      method: chapter.method.toString(),
+      method: chapter.method,
       headers: { 'Content-Type' : content_type, ...headers },
       params,
       data: request_data,

diff --git a/tools/src/tester/OperationLocator.ts b/tools/src/tester/OperationLocator.ts
@@ -9,9 +9,9 @@
 
 import { type OpenAPIV3 } from 'openapi-types'
 import { resolve_ref } from '../helpers'
-import { type Chapter } from './types/story.types'
 import { type ParsedOperation } from './types/spec.types'
 import _ from 'lodash'
+import { ParsedChapter } from './types/parsed_story.types'
 
 export default class OperationLocator {
   private readonly spec: OpenAPIV3.Document
@@ -21,9 +21,9 @@ export default class OperationLocator {
     this.spec = spec
   }
 
-  locate_operation (chapter: Chapter): ParsedOperation | undefined {
+  locate_operation (chapter: ParsedChapter): ParsedOperation | undefined {
     const path = chapter.path
-    const method = chapter.method.toString().toLowerCase() as OpenAPIV3.HttpMethods
+    const method = chapter.method.toLowerCase() as OpenAPIV3.HttpMethods
     const cache_key = path + method
     if (this.cached_operations[cache_key] != null) return this.cached_operations[cache_key]
     const operation = this.spec.paths[path]?.[method]

diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts
@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import { ChapterRequest, Parameter, type Chapter, type Story, type SupplementalChapter } from './types/story.types'
+import { Parameter, type Story } from './types/story.types'
 import { type StoryFile, type ChapterEvaluation, Result, type StoryEvaluation, OutputReference } from './types/eval.types'
 import type ChapterEvaluator from './ChapterEvaluator'
 import { overall_result } from './helpers'
@@ -16,6 +16,7 @@ import SupplementalChapterEvaluator from './SupplementalChapterEvaluator'
 import { ChapterOutput } from './ChapterOutput'
 import * as semver from '../_utils/semver'
 import _ from 'lodash'
+import { ParsedChapter, ParsedChapterRequest, ParsedStory, ParsedSupplementalChapter } from './types/parsed_story.types'
 
 export default class StoryEvaluator {
   private readonly _chapter_evaluator: ChapterEvaluator
@@ -105,20 +106,20 @@ export default class StoryEvaluator {
     }
   }
 
-  async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string, distribution?: string): Promise<ChapterEvaluation[]> {
+  async #evaluate_chapters(chapters: ParsedChapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string, distribution?: string): Promise<ChapterEvaluation[]> {
     const evaluations: ChapterEvaluation[] = []
     for (const chapter of chapters) {
       if (dry_run) {
-        const title = chapter.synopsis || `${chapter.method.toString()} ${chapter.path}`
+        const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } })
       } else if (distribution != undefined && chapter.distributions?.included !== undefined && chapter.distributions?.included.length > 0 && !chapter.distributions.included.includes(distribution)) {
-        const title = chapter.synopsis || `${chapter.method.toString()} ${chapter.path}`
+        const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.included.length > 1 ? 'one of ' : ''}${chapter.distributions.included.join(', ')}.` } })
       } else if (distribution != undefined && chapter.distributions?.excluded !== undefined && chapter.distributions?.excluded.length > 0 && chapter.distributions.excluded.includes(distribution)) {
-        const title = chapter.synopsis || `${chapter.method.toString()} ${chapter.path}`
+        const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is ${chapter.distributions.excluded.length > 1 ? 'one of ' : ''}${chapter.distributions.excluded.join(', ')}.` } })
       } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) {
-        const title = chapter.synopsis || `${chapter.method.toString()} ${chapter.path}`
+        const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.` } })
       } else {
         const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs)
@@ -132,11 +133,11 @@ export default class StoryEvaluator {
     return evaluations
   }
 
-  async #evaluate_supplemental_chapters(chapters: SupplementalChapter[], dry_run: boolean, story_outputs: StoryOutputs): Promise<{ evaluations: ChapterEvaluation[], has_errors: boolean }> {
+  async #evaluate_supplemental_chapters(chapters: ParsedSupplementalChapter[], dry_run: boolean, story_outputs: StoryOutputs): Promise<{ evaluations: ChapterEvaluation[], has_errors: boolean }> {
     let has_errors = false
     const evaluations: ChapterEvaluation[] = []
     for (const chapter of chapters) {
-      const title = `${chapter.method.toString()} ${chapter.path}`
+      const title = `${chapter.method} ${chapter.path}`
       if (dry_run) {
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } })
       } else {
@@ -152,7 +153,7 @@ export default class StoryEvaluator {
   }
 
   // TODO: Refactor and move this logic into StoryValidator
-  static check_story_variables(story: Story, display_path: string, full_path: string): StoryEvaluation | undefined {
+  static check_story_variables(story: ParsedStory, display_path: string, full_path: string): StoryEvaluation | undefined {
     const story_outputs = new StoryOutputs()
     const prologues = (story.prologues ?? []).map((prologue) => {
       return StoryEvaluator.#check_chapter_variables(prologue, story_outputs)
@@ -178,8 +179,8 @@ export default class StoryEvaluator {
     }
   }
 
-  static #check_chapter_variables(chapter: ChapterRequest, story_outputs: StoryOutputs): ChapterEvaluation {
-    const title = `${chapter.method.toString()} ${chapter.path}`
+  static #check_chapter_variables(chapter: ParsedChapterRequest, story_outputs: StoryOutputs): ChapterEvaluation {
+    const title = `${chapter.method} ${chapter.path}`
     const error = StoryEvaluator.#check_used_variables(chapter, story_outputs)
     if (error !== undefined) {
       return error
@@ -201,9 +202,9 @@ export default class StoryEvaluator {
    * @param story_outputs
    * @returns
    */
-  static #check_used_variables(chapter: ChapterRequest, story_outputs: StoryOutputs): ChapterEvaluation | undefined {
+  static #check_used_variables(chapter: ParsedChapterRequest, story_outputs: StoryOutputs): ChapterEvaluation | undefined {
     const variables = new Set<OutputReference>()
-    const title = `${chapter.method.toString()} ${chapter.path}`
+    const title = `${chapter.method} ${chapter.path}`
     StoryEvaluator.#extract_params_variables(chapter.parameters ?? {}, variables)
     StoryEvaluator.#extract_request_variables(chapter.request?.payload ?? {}, variables)
     for (const { chapter_id, output_name } of variables) {

diff --git a/tools/src/tester/StoryParser.ts b/tools/src/tester/StoryParser.ts
@@ -0,0 +1,39 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import _ from "lodash";
+import { ChapterRequest, Story } from "./types/story.types";
+import { ParsedChapter, ParsedChapterRequest, ParsedStory } from "./types/parsed_story.types";
+
+export default class StoryParser {
+  static parse(story: Story): ParsedStory {
+    return {
+      ...story,
+      chapters: this.#expand_chapters(story.chapters) as ParsedChapter[],
+      prologues: this.#expand_chapters(story.prologues),
+      epilogues: this.#expand_chapters(story.epilogues)
+    }
+  }
+
+  static #chapter_methods(methods: string[] | string): string[] {
+    return [...(Array.isArray(methods) ? methods : [methods])]
+  }
+
+  static #expand_chapters(chapters?: ChapterRequest[]): ParsedChapterRequest[] {
+    if (chapters === undefined) return []
+    return  _.flatMap(_.map(chapters, (chapter) => {
+      return _.map(this.#chapter_methods(chapter.method), (method) => {
+        return {
+          ...chapter,
+          method
+        }
+      })
+    })) as ParsedChapterRequest[]
+  }
+}
diff --git a/tools/src/tester/StoryReader.ts b/tools/src/tester/StoryReader.ts
diff --git a/tools/src/tester/SupplementalChapterEvaluator.ts b/tools/src/tester/SupplementalChapterEvaluator.ts
@@ -13,9 +13,9 @@ import ChapterReader from "./ChapterReader";
 import { StoryOutputs } from "./StoryOutputs";
 import { overall_result } from "./helpers";
 import { ChapterEvaluation, EvaluationWithOutput, Result } from './types/eval.types';
-import { SupplementalChapter } from "./types/story.types";
 import { Logger } from "../Logger";
 import { sleep, to_json } from "../helpers";
+import { ParsedSupplementalChapter } from "./types/parsed_story.types";
 
 export default class SupplementalChapterEvaluator {
   private readonly _chapter_reader: ChapterReader;
@@ -26,8 +26,8 @@ export default class SupplementalChapterEvaluator {
     this.logger = logger
   }
 
-  async evaluate(chapter: SupplementalChapter, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
-    const title = `${chapter.method.toString()} ${chapter.path}`
+  async evaluate(chapter: ParsedSupplementalChapter, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
+    const title = `${chapter.method} ${chapter.path}`
 
     let tries = chapter.retry && chapter.retry?.count > 0 ? chapter.retry.count + 1 : 1
     let chapter_evaluation: EvaluationWithOutput
@@ -45,7 +45,7 @@ export default class SupplementalChapterEvaluator {
     return result
   }
 
-  async #evaluate(chapter: SupplementalChapter, story_outputs: StoryOutputs): Promise<EvaluationWithOutput> {
+  async #evaluate(chapter: ParsedSupplementalChapter, story_outputs: StoryOutputs): Promise<EvaluationWithOutput> {
     const response = await this._chapter_reader.read(chapter, story_outputs)
     const output_values_evaluation = ChapterOutput.extract_output_values(response, chapter.output)
     if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`)