Use strong types for parsed stories.

Signed-off-by: dblock <[email protected]>
opensearch-project · Dec 11, 2024 · 22d446d · 22d446d
1 parent 5024aec
commit 22d446d
Show file tree

Hide file tree

Showing 13 changed files with 140 additions and 86 deletions.
diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts
@@ -21,6 +21,7 @@ import _ from 'lodash'
 import { Logger } from 'Logger'
 import { sleep, to_json } from '../helpers'
 import { APPLICATION_JSON } from "./MimeTypes";
+import { ParsedChapter } from './types/parsed_story.types'
 
 export default class ChapterEvaluator {
   private readonly logger: Logger
@@ -35,11 +36,11 @@ export default class ChapterEvaluator {
     this.logger = logger
   }
 
-  async evaluate(chapter: Chapter, skip: boolean, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
+  async evaluate(chapter: ParsedChapter, skip: boolean, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
     if (skip) return { title: chapter.synopsis, overall: { result: Result.SKIPPED } }
 
     const operation = this._operation_locator.locate_operation(chapter)
-    if (operation == null) return { title: chapter.synopsis, overall: { result: Result.FAILED, message: `Operation "${chapter.method.toString().toUpperCase()} ${chapter.path}" not found in the spec.` } }
+    if (operation == null) return { title: chapter.synopsis, overall: { result: Result.FAILED, message: `Operation "${chapter.method.toUpperCase()} ${chapter.path}" not found in the spec.` } }
 
     var tries = chapter.retry && chapter.retry?.count > 0 ? chapter.retry.count + 1 : 1
     var retry = 0
@@ -61,7 +62,7 @@ export default class ChapterEvaluator {
     return result
   }
 
-  async #evaluate(chapter: Chapter, operation: ParsedOperation, story_outputs: StoryOutputs, retries?: number): Promise<ChapterEvaluation> {
+  async #evaluate(chapter: ParsedChapter, operation: ParsedOperation, story_outputs: StoryOutputs, retries?: number): Promise<ChapterEvaluation> {
     const response = await this._chapter_reader.read(chapter, story_outputs)
     const params = this.#evaluate_parameters(chapter, operation, story_outputs)
     const request = this.#evaluate_request(chapter, operation, story_outputs)
@@ -85,10 +86,10 @@ export default class ChapterEvaluator {
     var result: ChapterEvaluation = {
       title: chapter.synopsis,
       operation: {
-        method: chapter.method.toString(),
+        method: chapter.method,
         path: chapter.path
       },
-      path: `${chapter.method.toString()} ${chapter.path}`,
+      path: `${chapter.method} ${chapter.path}`,
       overall: { result: overall_result(evaluations) },
       request: { parameters: params, request },
       response: {

diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts
@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import { type ChapterRequest, type ActualResponse, type Parameter } from './types/story.types'
+import { type ActualResponse, type Parameter } from './types/story.types'
 import { type OpenSearchHttpClient } from '../OpenSearchHttpClient'
 import { type StoryOutputs } from './StoryOutputs'
 import { Logger } from 'Logger'
@@ -18,6 +18,7 @@ import CBOR from 'cbor'
 import SMILE from 'smile-js'
 import { APPLICATION_CBOR, APPLICATION_JSON, APPLICATION_SMILE, APPLICATION_YAML, TEXT_PLAIN } from "./MimeTypes";
 import _ from 'lodash'
+import { ParsedChapterRequest } from './types/parsed_story.types'
 
 export default class ChapterReader {
   private readonly _client: OpenSearchHttpClient
@@ -28,7 +29,7 @@ export default class ChapterReader {
     this.logger = logger
   }
 
-  async read (chapter: ChapterRequest, story_outputs: StoryOutputs): Promise<ActualResponse> {
+  async read (chapter: ParsedChapterRequest, story_outputs: StoryOutputs): Promise<ActualResponse> {
     const response: Record<string, any> = {}
     const resolved_params = story_outputs.resolve_params(chapter.parameters ?? {})
     const [url_path, params] = this.#parse_url(chapter.path, resolved_params)
@@ -37,10 +38,10 @@ export default class ChapterReader {
       story_outputs.resolve_value(chapter.request.payload),
       content_type
     ) : undefined
-    this.logger.info(`=> ${chapter.method.toString()} ${url_path} (${to_json(params)}) [${content_type}] ${_.compact([to_json(headers), to_json(request_data)]).join(' | ')}`)
+    this.logger.info(`=> ${chapter.method} ${url_path} (${to_json(params)}) [${content_type}] ${_.compact([to_json(headers), to_json(request_data)]).join(' | ')}`)
     await this._client.request({
       url: url_path,
-      method: chapter.method.toString(),
+      method: chapter.method,
       headers: { 'Content-Type' : content_type, ...headers },
       params,
       data: request_data,

diff --git a/tools/src/tester/OperationLocator.ts b/tools/src/tester/OperationLocator.ts
@@ -9,9 +9,9 @@
 
 import { type OpenAPIV3 } from 'openapi-types'
 import { resolve_ref } from '../helpers'
-import { type Chapter } from './types/story.types'
 import { type ParsedOperation } from './types/spec.types'
 import _ from 'lodash'
+import { ParsedChapter } from './types/parsed_story.types'
 
 export default class OperationLocator {
   private readonly spec: OpenAPIV3.Document
@@ -21,9 +21,9 @@ export default class OperationLocator {
     this.spec = spec
   }
 
-  locate_operation (chapter: Chapter): ParsedOperation | undefined {
+  locate_operation (chapter: ParsedChapter): ParsedOperation | undefined {
     const path = chapter.path
-    const method = chapter.method.toString().toLowerCase() as OpenAPIV3.HttpMethods
+    const method = chapter.method.toLowerCase() as OpenAPIV3.HttpMethods
     const cache_key = path + method
     if (this.cached_operations[cache_key] != null) return this.cached_operations[cache_key]
     const operation = this.spec.paths[path]?.[method]

diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts
@@ -7,7 +7,7 @@
 * compatible open source license.
 */
 
-import { ChapterRequest, Parameter, type Chapter, type Story, type SupplementalChapter } from './types/story.types'
+import { Parameter, type Story } from './types/story.types'
 import { type StoryFile, type ChapterEvaluation, Result, type StoryEvaluation, OutputReference } from './types/eval.types'
 import type ChapterEvaluator from './ChapterEvaluator'
 import { overall_result } from './helpers'
@@ -16,6 +16,7 @@ import SupplementalChapterEvaluator from './SupplementalChapterEvaluator'
 import { ChapterOutput } from './ChapterOutput'
 import * as semver from '../_utils/semver'
 import _ from 'lodash'
+import { ParsedChapter, ParsedChapterRequest, ParsedStory, ParsedSupplementalChapter } from './types/parsed_story.types'
 
 export default class StoryEvaluator {
   private readonly _chapter_evaluator: ChapterEvaluator
@@ -105,20 +106,20 @@ export default class StoryEvaluator {
     }
   }
 
-  async #evaluate_chapters(chapters: Chapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string, distribution?: string): Promise<ChapterEvaluation[]> {
+  async #evaluate_chapters(chapters: ParsedChapter[], has_errors: boolean, dry_run: boolean, story_outputs: StoryOutputs, version?: string, distribution?: string): Promise<ChapterEvaluation[]> {
     const evaluations: ChapterEvaluation[] = []
     for (const chapter of chapters) {
       if (dry_run) {
-        const title = chapter.synopsis || `${chapter.method.toString()} ${chapter.path}`
+        const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } })
       } else if (distribution != undefined && chapter.distributions?.included !== undefined && chapter.distributions?.included.length > 0 && !chapter.distributions.included.includes(distribution)) {
-        const title = chapter.synopsis || `${chapter.method.toString()} ${chapter.path}`
+        const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is not ${chapter.distributions.included.length > 1 ? 'one of ' : ''}${chapter.distributions.included.join(', ')}.` } })
       } else if (distribution != undefined && chapter.distributions?.excluded !== undefined && chapter.distributions?.excluded.length > 0 && chapter.distributions.excluded.includes(distribution)) {
-        const title = chapter.synopsis || `${chapter.method.toString()} ${chapter.path}`
+        const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because distribution ${distribution} is ${chapter.distributions.excluded.length > 1 ? 'one of ' : ''}${chapter.distributions.excluded.join(', ')}.` } })
       } else if (version != undefined && chapter.version !== undefined && !semver.satisfies(version, chapter.version)) {
-        const title = chapter.synopsis || `${chapter.method.toString()} ${chapter.path}`
+        const title = chapter.synopsis || `${chapter.method} ${chapter.path}`
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: `Skipped because version ${version} does not satisfy ${chapter.version}.` } })
       } else {
         const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors, story_outputs)
@@ -132,11 +133,11 @@ export default class StoryEvaluator {
     return evaluations
   }
 
-  async #evaluate_supplemental_chapters(chapters: SupplementalChapter[], dry_run: boolean, story_outputs: StoryOutputs): Promise<{ evaluations: ChapterEvaluation[], has_errors: boolean }> {
+  async #evaluate_supplemental_chapters(chapters: ParsedSupplementalChapter[], dry_run: boolean, story_outputs: StoryOutputs): Promise<{ evaluations: ChapterEvaluation[], has_errors: boolean }> {
     let has_errors = false
     const evaluations: ChapterEvaluation[] = []
     for (const chapter of chapters) {
-      const title = `${chapter.method.toString()} ${chapter.path}`
+      const title = `${chapter.method} ${chapter.path}`
       if (dry_run) {
         evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run' } })
       } else {
@@ -152,7 +153,7 @@ export default class StoryEvaluator {
   }
 
   // TODO: Refactor and move this logic into StoryValidator
-  static check_story_variables(story: Story, display_path: string, full_path: string): StoryEvaluation | undefined {
+  static check_story_variables(story: ParsedStory, display_path: string, full_path: string): StoryEvaluation | undefined {
     const story_outputs = new StoryOutputs()
     const prologues = (story.prologues ?? []).map((prologue) => {
       return StoryEvaluator.#check_chapter_variables(prologue, story_outputs)
@@ -178,8 +179,8 @@ export default class StoryEvaluator {
     }
   }
 
-  static #check_chapter_variables(chapter: ChapterRequest, story_outputs: StoryOutputs): ChapterEvaluation {
-    const title = `${chapter.method.toString()} ${chapter.path}`
+  static #check_chapter_variables(chapter: ParsedChapterRequest, story_outputs: StoryOutputs): ChapterEvaluation {
+    const title = `${chapter.method} ${chapter.path}`
     const error = StoryEvaluator.#check_used_variables(chapter, story_outputs)
     if (error !== undefined) {
       return error
@@ -201,9 +202,9 @@ export default class StoryEvaluator {
    * @param story_outputs
    * @returns
    */
-  static #check_used_variables(chapter: ChapterRequest, story_outputs: StoryOutputs): ChapterEvaluation | undefined {
+  static #check_used_variables(chapter: ParsedChapterRequest, story_outputs: StoryOutputs): ChapterEvaluation | undefined {
     const variables = new Set<OutputReference>()
-    const title = `${chapter.method.toString()} ${chapter.path}`
+    const title = `${chapter.method} ${chapter.path}`
     StoryEvaluator.#extract_params_variables(chapter.parameters ?? {}, variables)
     StoryEvaluator.#extract_request_variables(chapter.request?.payload ?? {}, variables)
     for (const { chapter_id, output_name } of variables) {

diff --git a/tools/src/tester/StoryParser.ts b/tools/src/tester/StoryParser.ts
@@ -0,0 +1,39 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import _ from "lodash";
+import { ChapterRequest, Story } from "./types/story.types";
+import { ParsedChapter, ParsedChapterRequest, ParsedStory } from "./types/parsed_story.types";
+
+export default class StoryParser {
+  static parse(story: Story): ParsedStory {
+    return {
+      ...story,
+      chapters: this.#expand_chapters(story.chapters) as ParsedChapter[],
+      prologues: this.#expand_chapters(story.prologues),
+      epilogues: this.#expand_chapters(story.epilogues)
+    }
+  }
+
+  static #chapter_methods(methods: string[] | string): string[] {
+    return [...(Array.isArray(methods) ? methods : [methods])]
+  }
+
+  static #expand_chapters(chapters?: ChapterRequest[]): ParsedChapterRequest[] {
+    if (chapters === undefined) return []
+    return  _.flatMap(_.map(chapters, (chapter) => {
+      return _.map(this.#chapter_methods(chapter.method), (method) => {
+        return {
+          ...chapter,
+          method
+        }
+      })
+    })) as ParsedChapterRequest[]
+  }
+}
diff --git a/tools/src/tester/StoryReader.ts b/tools/src/tester/StoryReader.ts
diff --git a/tools/src/tester/SupplementalChapterEvaluator.ts b/tools/src/tester/SupplementalChapterEvaluator.ts
@@ -13,9 +13,9 @@ import ChapterReader from "./ChapterReader";
 import { StoryOutputs } from "./StoryOutputs";
 import { overall_result } from "./helpers";
 import { ChapterEvaluation, EvaluationWithOutput, Result } from './types/eval.types';
-import { SupplementalChapter } from "./types/story.types";
 import { Logger } from "../Logger";
 import { sleep, to_json } from "../helpers";
+import { ParsedSupplementalChapter } from "./types/parsed_story.types";
 
 export default class SupplementalChapterEvaluator {
   private readonly _chapter_reader: ChapterReader;
@@ -26,8 +26,8 @@ export default class SupplementalChapterEvaluator {
     this.logger = logger
   }
 
-  async evaluate(chapter: SupplementalChapter, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
-    const title = `${chapter.method.toString()} ${chapter.path}`
+  async evaluate(chapter: ParsedSupplementalChapter, story_outputs: StoryOutputs): Promise<ChapterEvaluation> {
+    const title = `${chapter.method} ${chapter.path}`
 
     let tries = chapter.retry && chapter.retry?.count > 0 ? chapter.retry.count + 1 : 1
     let chapter_evaluation: EvaluationWithOutput
@@ -45,7 +45,7 @@ export default class SupplementalChapterEvaluator {
     return result
   }
 
-  async #evaluate(chapter: SupplementalChapter, story_outputs: StoryOutputs): Promise<EvaluationWithOutput> {
+  async #evaluate(chapter: ParsedSupplementalChapter, story_outputs: StoryOutputs): Promise<EvaluationWithOutput> {
     const response = await this._chapter_reader.read(chapter, story_outputs)
     const output_values_evaluation = ChapterOutput.extract_output_values(response, chapter.output)
     if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`)

diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts
@@ -10,7 +10,6 @@
 import type StoryEvaluator from './StoryEvaluator'
 import { StoryEvaluations, type StoryFile } from './types/eval.types'
 import fs from 'fs'
-import { type Story } from './types/story.types'
 import { read_yaml } from '../helpers'
 import { Result } from './types/eval.types'
 import { type ResultLogger } from './ResultLogger'
@@ -20,7 +19,7 @@ import { OpenSearchHttpClient } from 'OpenSearchHttpClient'
 import * as ansi from './Ansi'
 import _ from 'lodash'
 import { Logger } from 'Logger'
-import StoryReader from './StoryReader'
+import StoryParser from './StoryParser'
 
 export default class TestRunner {
   private readonly _http_client: OpenSearchHttpClient
@@ -56,9 +55,8 @@ export default class TestRunner {
     }
 
     for (const story_file of story_files) {
-      var story_reader = new StoryReader(story_file)
-      this._logger.info(`Evaluating ${story_reader.display_path} ...`)
-      const evaluation = this._story_validator.validate(story_reader.story_file) ?? await this._story_evaluator.evaluate(story_reader.story_file, version, distribution, dry_run)
+      this._logger.info(`Evaluating ${story_file.display_path} ...`)
+      const evaluation = this._story_validator.validate(story_file) ?? await this._story_evaluator.evaluate(story_file, version, distribution, dry_run)
       results.evaluations.push(evaluation)
       this._result_logger.log(evaluation)
       if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true
@@ -79,7 +77,7 @@ export default class TestRunner {
     if (file.startsWith('.') || file == 'docker-compose.yml' || file == 'Dockerfile' || file.endsWith('.py')) {
       return []
     } else if (fs.statSync(path).isFile()) {
-      const story: Story = read_yaml(path)
+      const story = StoryParser.parse(read_yaml(path))
       return [{
         display_path: next_prefix === '' ? basename(path) : next_prefix,
         full_path: path,

diff --git a/tools/src/tester/types/eval.types.ts b/tools/src/tester/types/eval.types.ts
@@ -9,12 +9,12 @@
 
 import { type ChapterOutput } from '../ChapterOutput'
 import { StoryOutputs } from '../StoryOutputs'
-import type { Story } from "./story.types";
+import { ParsedStory } from './parsed_story.types';
 
 export interface StoryFile {
   display_path: string
   full_path: string
-  story: Story
+  story: ParsedStory
 }
 
 export interface Operation {

diff --git a/tools/src/tester/types/parsed_story.types.ts b/tools/src/tester/types/parsed_story.types.ts
@@ -0,0 +1,23 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import { Chapter, ChapterRequest, HttpMethod, Story, SupplementalChapter } from "./story.types"
+
+export interface ParsedChapterRequest extends ChapterRequest {
+  method: HttpMethod
+}
+
+export type ParsedChapter = ParsedChapterRequest & Chapter
+export type ParsedSupplementalChapter = ParsedChapterRequest & SupplementalChapter
+
+export interface ParsedStory extends Story {
+  chapters: ParsedChapter[]
+  prologues?: ParsedSupplementalChapter[]
+  epilogues?: ParsedSupplementalChapter[]
+}