diff --git a/.github/workflows/analyze-pr-changes.yml b/.github/workflows/analyze-pr-changes.yml index ee2cc06ee..14bbb1ed2 100644 --- a/.github/workflows/analyze-pr-changes.yml +++ b/.github/workflows/analyze-pr-changes.yml @@ -59,7 +59,7 @@ jobs: npm install - npm run dump-cluster-spec -- --insecure --output $CLUSTER_SPEC + npm run dump-cluster-spec -- --opensearch-insecure --output $CLUSTER_SPEC docker stop opensearch env: diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index cc3826300..e49eb534d 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -22,7 +22,6 @@ jobs: env: OPENSEARCH_VERSION: 2.12.0 OPENSEARCH_PASSWORD: myStrongPassword123! - OPENSEARCH_URL: https://localhost:9200 steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -37,9 +36,7 @@ jobs: - name: Run OpenSearch Cluster working-directory: .github/opensearch-cluster - run: | - docker-compose up -d - sleep 60 + run: docker-compose up -d && sleep 60 - name: Run Tests - run: npm run test:spec \ No newline at end of file + run: npm run test:spec -- --opensearch-insecure \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index fa76a054a..ef2cbacbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Changed - Replaced Smithy with a native OpenAPI spec ([#189](https://github.com/opensearch-project/opensearch-api-specification/issues/189)) +- Refactored spec tester internals to improve reusability ([#302](https://github.com/opensearch-project/opensearch-api-specification/pull/302)) ### Deprecated diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 22a1d5e25..65229fae4 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -284,12 +284,10 @@ The dump-cluster-spec tool connects to an OpenSearch cluster which has the [open #### Arguments -- `--host `: The host at which the cluster is accessible, defaults to `localhost`. -- `--port `: The port at which the cluster is accessible, defaults to `9200`. -- `--no-https`: Disable HTTPS, defaults to using HTTPS. -- `--insecure`: Disable SSL/TLS certificate verification, defaults to performing verification. -- `--username `: The username to authenticate with the cluster, defaults to `admin`, only used when `--password` is set. -- `--password `: The password to authenticate with the cluster, also settable via the `OPENSEARCH_PASSWORD` environment variable. +- `--opensearch-url `: The URL at which the cluster is accessible, defaults to `https://localhost:9200`. +- `--opensearch-insecure`: Disable SSL/TLS certificate verification, defaults to performing verification. +- `--opensearch-username `: The username to authenticate with the cluster, defaults to `admin`, only used when `--opensearch-password` is set. +- `--opensearch-password `: The password to authenticate with the cluster, also settable via the `OPENSEARCH_PASSWORD` environment variable. - `--output `: The path to write the dumped spec to, defaults to `/build/opensearch-openapi-CLUSTER.yaml`. #### Example @@ -308,7 +306,7 @@ docker run \ -e OPENSEARCH_INITIAL_ADMIN_PASSWORD="$OPENSEARCH_PASSWORD" \ opensearch-with-api-plugin -OPENSEARCH_PASSWORD="${OPENSEARCH_PASSWORD}" npm run dump-cluster-spec -- --insecure +OPENSEARCH_PASSWORD="${OPENSEARCH_PASSWORD}" npm run dump-cluster-spec -- --opensearch-insecure docker stop opensearch ``` diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts new file mode 100644 index 000000000..76cb8672d --- /dev/null +++ b/tools/src/OpenSearchHttpClient.ts @@ -0,0 +1,132 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { Option } from '@commander-js/extra-typings' +import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse } from 'axios' +import * as https from 'node:https' +import { sleep } from '../helpers' + +const DEFAULT_URL = 'https://localhost:9200' +const DEFAULT_USER = 'admin' +const DEFAULT_INSECURE = false + +export const OPENSEARCH_URL_OPTION = new Option('--opensearch-url ', 'URL at which the OpenSearch cluster is accessible') + .default(DEFAULT_URL) + .env('OPENSEARCH_URL') + +export const OPENSEARCH_USERNAME_OPTION = new Option('--opensearch-username ', 'username to use when authenticating with OpenSearch') + .default(DEFAULT_USER) + .env('OPENSEARCH_USERNAME') + +export const OPENSEARCH_PASSWORD_OPTION = new Option('--opensearch-password ', 'password to use when authenticating with OpenSearch') + .env('OPENSEARCH_PASSWORD') + +export const OPENSEARCH_INSECURE_OPTION = new Option('--opensearch-insecure', 'disable SSL/TLS certificate verification when connecting to OpenSearch') + .default(DEFAULT_INSECURE) + +export interface OpenSearchHttpClientOptions { + url?: string + username?: string + password?: string + insecure?: boolean +} + +export type OpenSearchHttpClientCliOptions = { [K in keyof OpenSearchHttpClientOptions as `opensearch${Capitalize}`]: OpenSearchHttpClientOptions[K] } + +export function get_opensearch_opts_from_cli (opts: OpenSearchHttpClientCliOptions): OpenSearchHttpClientOptions { + return { + url: opts.opensearchUrl, + username: opts.opensearchUsername, + password: opts.opensearchPassword, + insecure: opts.opensearchInsecure + } +} + +export interface OpenSearchInfo { + cluster_name: string + cluster_uuid: string + name: string + tagline: string + version: { + build_date: string + build_flavor: string + build_hash: string + build_snapshot: boolean + build_type: string + lucene_version: string + minimum_index_compatibility_version: string + minimum_wire_compatibility_version: string + number: string + } +} + +export class OpenSearchHttpClient { + private readonly _axios: AxiosInstance + + constructor (opts?: OpenSearchHttpClientOptions) { + this._axios = axios.create({ + baseURL: opts?.url ?? DEFAULT_URL, + auth: opts?.username !== undefined && opts.password !== undefined + ? { + username: opts.username, + password: opts.password + } + : undefined, + httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }) + }) + } + + async wait_until_available (max_attempts: number = 20, wait_between_attempt_millis: number = 5000): Promise { + let attempt = 0 + while (true) { + attempt += 1 + try { + const info = await this.get('/') + return info.data + } catch (e) { + if (attempt >= max_attempts) { + throw e + } + await sleep(wait_between_attempt_millis) + } + } + } + + async request, D = any>(config: AxiosRequestConfig): Promise { + return await this._axios.request(config) + } + + async get, D = any>(url: string, config?: AxiosRequestConfig): Promise { + return await this._axios.get(url, config) + } + + async delete, D = any>(url: string, config?: AxiosRequestConfig): Promise { + return await this._axios.delete(url, config) + } + + async head, D = any>(url: string, config?: AxiosRequestConfig): Promise { + return await this._axios.head(url, config) + } + + async options, D = any>(url: string, config?: AxiosRequestConfig): Promise { + return await this._axios.options(url, config) + } + + async post, D = any>(url: string, data?: D, config?: AxiosRequestConfig): Promise { + return await this._axios.post(url, data, config) + } + + async put, D = any>(url: string, data?: D, config?: AxiosRequestConfig): Promise { + return await this._axios.put(url, data, config) + } + + async patch, D = any>(url: string, data?: D, config?: AxiosRequestConfig): Promise { + return await this._axios.patch(url, data, config) + } +} diff --git a/tools/src/dump-cluster-spec/dump-cluster-spec.ts b/tools/src/dump-cluster-spec/dump-cluster-spec.ts index bb7bd96de..16038396c 100644 --- a/tools/src/dump-cluster-spec/dump-cluster-spec.ts +++ b/tools/src/dump-cluster-spec/dump-cluster-spec.ts @@ -9,68 +9,46 @@ import { Command, Option } from '@commander-js/extra-typings' import { resolve } from 'path' -import axios from 'axios' -import * as https from 'node:https' import * as process from 'node:process' -import { sleep, write_yaml } from '../../helpers' +import { write_yaml } from '../../helpers' +import { + get_opensearch_opts_from_cli, + OPENSEARCH_INSECURE_OPTION, + OPENSEARCH_PASSWORD_OPTION, + OPENSEARCH_URL_OPTION, + OPENSEARCH_USERNAME_OPTION, OpenSearchHttpClient, + type OpenSearchHttpClientOptions +} from '../OpenSearchHttpClient' interface CommandOpts { - host: string - https: boolean - insecure: boolean - port: string | number - username: string - password?: string + opensearch: OpenSearchHttpClientOptions output: string } async function main (opts: CommandOpts): Promise { - const url = `http${opts.https ? 's' : ''}://${opts.host}:${opts.port}` - const client = axios.create({ - httpsAgent: new https.Agent({ - rejectUnauthorized: !opts.insecure - }), - auth: opts.password !== undefined - ? { - username: opts.username, - password: opts.password - } - : undefined - }) + const client = new OpenSearchHttpClient(opts.opensearch) - let attempt = 0 - while (true) { - attempt += 1 - try { - const info = await client.get(url) - console.log(info.data) - break - } catch (e) { - if (attempt >= 20) { - throw e - } - await sleep(5000) - } - } + const info = await client.wait_until_available() + console.log(info) - const cluster_spec = await client.get(`${url}/_plugins/api`) + const cluster_spec = await client.get('/_plugins/api') write_yaml(opts.output, cluster_spec.data) } const command = new Command() .description('Dumps an OpenSearch cluster\'s generated specification.') - .addOption(new Option('--host ', 'cluster\'s host').default('localhost')) - .addOption(new Option('--no-https', 'disable HTTPS')) - .addOption(new Option('--insecure', 'disable SSL/TLS certificate verification').default(false)) - .addOption(new Option('--port ', 'cluster\'s port to connect to').default(9200)) - .addOption(new Option('--username ', 'username to authenticate with the cluster').default('admin')) - .addOption(new Option('--password ', 'password to authenticate with the cluster').env('OPENSEARCH_PASSWORD')) + .addOption(OPENSEARCH_URL_OPTION) + .addOption(OPENSEARCH_USERNAME_OPTION) + .addOption(OPENSEARCH_PASSWORD_OPTION) + .addOption(OPENSEARCH_INSECURE_OPTION) .addOption(new Option('--output ', 'path to the output file').default(resolve(__dirname, '../../../build/opensearch-openapi-CLUSTER.yaml'))) .allowExcessArguments(false) .parse() -main(command.opts()) +const opts = command.opts() + +main({ output: opts.output, opensearch: get_opensearch_opts_from_cli(opts) }) .catch(e => { if (e instanceof Error) { console.error(`ERROR: ${e.stack}`) diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index 333eb7840..c68b03d8e 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -12,62 +12,57 @@ import { type ChapterEvaluation, type Evaluation, Result } from './types/eval.ty import { type ParsedOperation } from './types/spec.types' import { overall_result } from './helpers' import type ChapterReader from './ChapterReader' -import SharedResources from './SharedResources' -import type SpecParser from './SpecParser' +import type OperationLocator from './OperationLocator' import type SchemaValidator from './SchemaValidator' export default class ChapterEvaluator { - chapter: Chapter - skip_payload_evaluation: boolean = false - spec_parser: SpecParser - chapter_reader: ChapterReader - schema_validator: SchemaValidator + private readonly _operation_locator: OperationLocator + private readonly _chapter_reader: ChapterReader + private readonly _schema_validator: SchemaValidator - constructor (chapter: Chapter) { - this.chapter = chapter - this.spec_parser = SharedResources.get_instance().spec_parser - this.chapter_reader = SharedResources.get_instance().chapter_reader - this.schema_validator = SharedResources.get_instance().schema_validator + constructor (spec_parser: OperationLocator, chapter_reader: ChapterReader, schema_validator: SchemaValidator) { + this._operation_locator = spec_parser + this._chapter_reader = chapter_reader + this._schema_validator = schema_validator } - async evaluate (skip: boolean): Promise { - if (skip) return { title: this.chapter.synopsis, overall: { result: Result.SKIPPED } } - const response = await this.chapter_reader.read(this.chapter) - const operation = this.spec_parser.locate_operation(this.chapter) - if (operation == null) return { title: this.chapter.synopsis, overall: { result: Result.FAILED, message: `Operation "${this.chapter.method.toUpperCase()} ${this.chapter.path}" not found in the spec.` } } - const params = this.#evaluate_parameters(operation) - const request_body = this.#evaluate_request_body(operation) - const status = this.#evaluate_status(response) - const payload = this.#evaluate_payload(operation, response) + async evaluate (chapter: Chapter, skip: boolean): Promise { + if (skip) return { title: chapter.synopsis, overall: { result: Result.SKIPPED } } + const response = await this._chapter_reader.read(chapter) + const operation = this._operation_locator.locate_operation(chapter) + if (operation == null) return { title: chapter.synopsis, overall: { result: Result.FAILED, message: `Operation "${chapter.method.toUpperCase()} ${chapter.path}" not found in the spec.` } } + const params = this.#evaluate_parameters(chapter, operation) + const request_body = this.#evaluate_request_body(chapter, operation) + const status = this.#evaluate_status(chapter, response) + const payload = status.result === Result.PASSED ? this.#evaluate_payload(response, operation) : { result: Result.SKIPPED } return { - title: this.chapter.synopsis, + title: chapter.synopsis, overall: { result: overall_result(Object.values(params).concat([request_body, status, payload])) }, request: { parameters: params, request_body }, response: { status, payload } } } - #evaluate_parameters (operation: ParsedOperation): Record { - return Object.fromEntries(Object.entries(this.chapter.parameters ?? {}).map(([name, parameter]) => { + #evaluate_parameters (chapter: Chapter, operation: ParsedOperation): Record { + return Object.fromEntries(Object.entries(chapter.parameters ?? {}).map(([name, parameter]) => { const schema = operation.parameters[name]?.schema if (schema == null) return [name, { result: Result.FAILED, message: `Schema for "${name}" parameter not found.` }] - const evaluation = this.schema_validator.validate(schema, parameter) + const evaluation = this._schema_validator.validate(schema, parameter) return [name, evaluation] })) } - #evaluate_request_body (operation: ParsedOperation): Evaluation { - if (!this.chapter.request_body) return { result: Result.PASSED } - const content_type = this.chapter.request_body.content_type ?? 'application/json' + #evaluate_request_body (chapter: Chapter, operation: ParsedOperation): Evaluation { + if (!chapter.request_body) return { result: Result.PASSED } + const content_type = chapter.request_body.content_type ?? 'application/json' const schema = operation.requestBody?.content[content_type]?.schema if (schema == null) return { result: Result.FAILED, message: `Schema for "${content_type}" request body not found in the spec.` } - return this.schema_validator.validate(schema, this.chapter.request_body?.payload ?? {}) + return this._schema_validator.validate(schema, chapter.request_body?.payload ?? {}) } - #evaluate_status (response: ActualResponse): Evaluation { - const expected_status = this.chapter.response?.status ?? 200 + #evaluate_status (chapter: Chapter, response: ActualResponse): Evaluation { + const expected_status = chapter.response?.status ?? 200 if (response.status === expected_status) return { result: Result.PASSED } - this.skip_payload_evaluation = true return { result: Result.ERROR, message: `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}. ${response.message}`, @@ -75,13 +70,12 @@ export default class ChapterEvaluator { } } - #evaluate_payload (operation: ParsedOperation, response: ActualResponse): Evaluation { - if (this.skip_payload_evaluation) return { result: Result.SKIPPED } + #evaluate_payload (response: ActualResponse, operation: ParsedOperation): Evaluation { const content_type = response.content_type ?? 'application/json' const content = operation.responses[response.status]?.content[content_type] const schema = content?.schema if (schema == null && content != null) return { result: Result.PASSED } if (schema == null) return { result: Result.FAILED, message: `Schema for "${response.status}: ${response.content_type}" response not found in the spec.` } - return this.schema_validator.validate(schema, response.payload) + return this._schema_validator.validate(schema, response.payload) } } diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts index 445508328..bd5774011 100644 --- a/tools/src/tester/ChapterReader.ts +++ b/tools/src/tester/ChapterReader.ts @@ -7,31 +7,22 @@ * compatible open source license. */ -import axios from 'axios' import { type ChapterRequest, type ActualResponse, type Parameter } from './types/story.types' -import { Agent } from 'https' +import { type OpenSearchHttpClient } from '../OpenSearchHttpClient' // A lightweight client for testing the API export default class ChapterReader { - url: string - admin_password: string + private readonly _client: OpenSearchHttpClient - constructor () { - this.url = process.env.OPENSEARCH_URL ?? 'https://localhost:9200' - if (process.env.OPENSEARCH_PASSWORD == null) throw new Error('OPENSEARCH_PASSWORD is not set') - this.admin_password = process.env.OPENSEARCH_PASSWORD + constructor (client: OpenSearchHttpClient) { + this._client = client } async read (chapter: ChapterRequest): Promise { const response: Record = {} - const [url, params] = this.#parse_url(chapter.path, chapter.parameters ?? {}) - await axios.request({ - url, - auth: { - username: 'admin', - password: this.admin_password - }, - httpsAgent: new Agent({ rejectUnauthorized: false }), + const [url_path, params] = this.#parse_url(chapter.path, chapter.parameters ?? {}) + await this._client.request({ + url: url_path, method: chapter.method, params, data: chapter.request_body?.payload @@ -57,7 +48,6 @@ export default class ChapterReader { return parameters[key] as string }) const query_params = Object.fromEntries(Object.entries(parameters).filter(([key]) => !path_params.has(key))) - const url = this.url + parsed_path - return [url, query_params] + return [parsed_path, query_params] } } diff --git a/tools/src/tester/SpecParser.ts b/tools/src/tester/OperationLocator.ts similarity index 98% rename from tools/src/tester/SpecParser.ts rename to tools/src/tester/OperationLocator.ts index 748642009..2d5df08ee 100644 --- a/tools/src/tester/SpecParser.ts +++ b/tools/src/tester/OperationLocator.ts @@ -13,7 +13,7 @@ import { type Chapter } from './types/story.types' import { type ParsedOperation } from './types/spec.types' import _ from 'lodash' -export default class SpecParser { +export default class OperationLocator { private readonly spec: OpenAPIV3.Document private cached_operations: Record = {} diff --git a/tools/src/tester/ResultLogger.ts b/tools/src/tester/ResultLogger.ts new file mode 100644 index 000000000..086387e81 --- /dev/null +++ b/tools/src/tester/ResultLogger.ts @@ -0,0 +1,104 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { type ChapterEvaluation, type Evaluation, Result, type StoryEvaluation } from './types/eval.types' +import { overall_result } from './helpers' +import * as ansi from './Ansi' + +export interface ResultLogger { + log: (evaluation: StoryEvaluation) => void +} + +export class NoOpResultLogger implements ResultLogger { + log (_: StoryEvaluation): void { } +} + +export class ConsoleResultLogger implements ResultLogger { + private readonly _tab_width: number + private readonly _verbose: boolean + + constructor (tab_width: number = 4, verbose: boolean = false) { + this._tab_width = tab_width + this._verbose = verbose + } + + log (evaluation: StoryEvaluation): void { + this.#log_story(evaluation) + this.#log_chapters(evaluation.prologues ?? [], 'PROLOGUES') + this.#log_chapters(evaluation.chapters ?? [], 'CHAPTERS') + this.#log_chapters(evaluation.epilogues ?? [], 'EPILOGUES') + console.log('\n') + } + + #log_story ({ result, full_path, description, display_path }: StoryEvaluation): void { + this.#log_evaluation({ result, message: full_path }, ansi.cyan(ansi.b(description ?? display_path))) + } + + #log_chapters (evaluations: ChapterEvaluation[], title: string): void { + if (evaluations.length === 0) return + const result = overall_result(evaluations.map(e => e.overall)) + if (!this._verbose && (result === Result.SKIPPED || result === Result.PASSED)) return + this.#log_evaluation({ result }, title, this._tab_width) + for (const evaluation of evaluations) this.#log_chapter(evaluation) + } + + #log_chapter (chapter: ChapterEvaluation): void { + this.#log_evaluation(chapter.overall, ansi.i(chapter.title), this._tab_width * 2) + this.#log_parameters(chapter.request?.parameters ?? {}) + this.#log_request_body(chapter.request?.request_body) + this.#log_status(chapter.response?.status) + this.#log_payload(chapter.response?.payload) + } + + #log_parameters (parameters: Record): void { + if (Object.keys(parameters).length === 0) return + const result = overall_result(Object.values(parameters)) + this.#log_evaluation({ result }, 'PARAMETERS', this._tab_width * 3) + for (const [name, evaluation] of Object.entries(parameters)) { + this.#log_evaluation(evaluation, name, this._tab_width * 4) + } + } + + #log_request_body (evaluation: Evaluation | undefined): void { + if (evaluation == null) return + this.#log_evaluation(evaluation, 'REQUEST BODY', this._tab_width * 3) + } + + #log_status (evaluation: Evaluation | undefined): void { + if (evaluation == null) return + this.#log_evaluation(evaluation, 'RESPONSE STATUS', this._tab_width * 3) + } + + #log_payload (evaluation: Evaluation | undefined): void { + if (evaluation == null) return + this.#log_evaluation(evaluation, 'RESPONSE PAYLOAD', this._tab_width * 3) + } + + #log_evaluation (evaluation: Evaluation, title: string, prefix: number = 0): void { + const result = ansi.padding(this.#result(evaluation.result), 0, prefix) + const message = evaluation.message != null ? `${ansi.gray('(' + evaluation.message + ')')}` : '' + console.log(`${result} ${title} ${message}`) + if (evaluation.error != null && this._verbose) { + console.log('-'.repeat(100)) + console.error(evaluation.error) + console.log('-'.repeat(100)) + } + } + + #result (r: Result): string { + const text = ansi.padding(r, 7) + switch (r) { + case Result.PASSED: return ansi.green(text) + case Result.SKIPPED: return ansi.yellow(text) + case Result.FAILED: return ansi.magenta(text) + case Result.ERROR: return ansi.red(text) + default: return ansi.gray(text) + } + } +} diff --git a/tools/src/tester/ResultsDisplayer.ts b/tools/src/tester/ResultsDisplayer.ts deleted file mode 100644 index a22353da9..000000000 --- a/tools/src/tester/ResultsDisplayer.ts +++ /dev/null @@ -1,110 +0,0 @@ -/* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ - -import { type ChapterEvaluation, type Evaluation, Result, type StoryEvaluation } from './types/eval.types' -import { overall_result } from './helpers' -import * as ansi from './Ansi' - -export interface TestRunOptions { - dry_run?: boolean -} - -export interface DisplayOptions { - tab_width?: number - verbose?: boolean -} - -export default class ResultsDisplayer { - evaluation: StoryEvaluation - tab_width: number - verbose: boolean - - constructor (evaluation: StoryEvaluation, opts: DisplayOptions) { - this.evaluation = evaluation - this.tab_width = opts.tab_width ?? 4 - this.verbose = opts.verbose ?? false - } - - display (): void { - this.#display_story() - this.#display_chapters(this.evaluation.prologues ?? [], 'PROLOGUES') - this.#display_chapters(this.evaluation.chapters ?? [], 'CHAPTERS') - this.#display_chapters(this.evaluation.epilogues ?? [], 'EPILOGUES') - console.log('\n') - } - - #display_story (): void { - const result = this.evaluation.result - const message = this.evaluation.full_path - const title = ansi.cyan(ansi.b(this.evaluation.description ?? this.evaluation.display_path)) - this.#display_evaluation({ result, message }, title) - } - - #display_chapters (evaluations: ChapterEvaluation[], title: string): void { - if (evaluations.length === 0) return - const result = overall_result(evaluations.map(e => e.overall)) - if (!this.verbose && (result === Result.SKIPPED || result === Result.PASSED)) return - this.#display_evaluation({ result }, title, this.tab_width) - for (const evaluation of evaluations) this.#display_chapter(evaluation) - } - - #display_chapter (chapter: ChapterEvaluation): void { - this.#display_evaluation(chapter.overall, ansi.i(chapter.title), this.tab_width * 2) - this.#display_parameters(chapter.request?.parameters ?? {}) - this.#display_request_body(chapter.request?.request_body) - this.#display_status(chapter.response?.status) - this.#display_payload(chapter.response?.payload) - } - - #display_parameters (parameters: Record): void { - if (Object.keys(parameters).length === 0) return - const result = overall_result(Object.values(parameters)) - this.#display_evaluation({ result }, 'PARAMETERS', this.tab_width * 3) - for (const [name, evaluation] of Object.entries(parameters)) { - this.#display_evaluation(evaluation, name, this.tab_width * 4) - } - } - - #display_request_body (evaluation: Evaluation | undefined): void { - if (evaluation == null) return - this.#display_evaluation(evaluation, 'REQUEST BODY', this.tab_width * 3) - } - - #display_status (evaluation: Evaluation | undefined): void { - if (evaluation == null) return - this.#display_evaluation(evaluation, 'RESPONSE STATUS', this.tab_width * 3) - } - - #display_payload (evaluation: Evaluation | undefined): void { - if (evaluation == null) return - this.#display_evaluation(evaluation, 'RESPONSE PAYLOAD', this.tab_width * 3) - } - - #display_evaluation (evaluation: Evaluation, title: string, prefix: number = 0): void { - const result = ansi.padding(this.#result(evaluation.result), 0, prefix) - const message = evaluation.message != null ? `${ansi.gray('(' + evaluation.message + ')')}` : '' - console.log(`${result} ${title} ${message}`) - if (evaluation.error && this.verbose) { - console.log('-'.repeat(100)) - console.error(evaluation.error) - console.log('-'.repeat(100)) - } - } - - #result (r: Result): string { - const text = ansi.padding(r, 7) - switch (r) { - case Result.PASSED: return ansi.green(text) - case Result.SKIPPED: return ansi.yellow(text) - case Result.FAILED: return ansi.magenta(text) - case Result.ERROR: return ansi.red(text) - default: return ansi.gray(text) - } - } -} diff --git a/tools/src/tester/SharedResources.ts b/tools/src/tester/SharedResources.ts deleted file mode 100644 index cf56dc87e..000000000 --- a/tools/src/tester/SharedResources.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ - -import type ChapterReader from './ChapterReader' -import type SchemaValidator from './SchemaValidator' -import type SpecParser from './SpecParser' - -interface Resources { - chapter_reader: ChapterReader - schema_validator: SchemaValidator - spec_parser: SpecParser -} - -export default class SharedResources { - private static instance: SharedResources | undefined - chapter_reader: ChapterReader - schema_validator: SchemaValidator - spec_parser: SpecParser - - private constructor (resources: Resources) { - this.chapter_reader = resources.chapter_reader - this.schema_validator = resources.schema_validator - this.spec_parser = resources.spec_parser - } - - public static create_instance (resources: Resources): void { - if (SharedResources.instance) throw new Error('SharedResources instance has already been created.') - SharedResources.instance = new SharedResources(resources) - } - - public static get_instance (): SharedResources { - if (SharedResources.instance) return SharedResources.instance - throw new Error('SharedResources instance has not been created.') - } -} diff --git a/tools/src/tester/StoryEvaluator.ts b/tools/src/tester/StoryEvaluator.ts index bafe2db43..9769195ac 100644 --- a/tools/src/tester/StoryEvaluator.ts +++ b/tools/src/tester/StoryEvaluator.ts @@ -9,9 +9,8 @@ import { type Chapter, type Story, type SupplementalChapter } from './types/story.types' import { type ChapterEvaluation, Result, type StoryEvaluation } from './types/eval.types' -import ChapterEvaluator from './ChapterEvaluator' +import type ChapterEvaluator from './ChapterEvaluator' import type ChapterReader from './ChapterReader' -import SharedResources from './SharedResources' import { overall_result } from './helpers' export interface StoryFile { @@ -21,38 +20,31 @@ export interface StoryFile { } export default class StoryEvaluator { - dry_run: boolean - story: Story - display_path: string - full_path: string - has_errors: boolean = false - chapter_reader: ChapterReader + private readonly _chapter_reader: ChapterReader + private readonly _chapter_evaluator: ChapterEvaluator - constructor (story_file: StoryFile, dry_run?: boolean) { - this.dry_run = dry_run ?? false - this.story = story_file.story - this.display_path = story_file.display_path - this.full_path = story_file.full_path - this.chapter_reader = SharedResources.get_instance().chapter_reader + constructor (chapter_reader: ChapterReader, chapter_evaluator: ChapterEvaluator) { + this._chapter_reader = chapter_reader + this._chapter_evaluator = chapter_evaluator } - async evaluate (): Promise { - if (this.story.skip) { + async evaluate ({ story, display_path, full_path }: StoryFile, dry_run: boolean = false): Promise { + if (story.skip) { return { result: Result.SKIPPED, - display_path: this.display_path, - full_path: this.full_path, - description: this.story.description, + display_path, + full_path, + description: story.description, chapters: [] } } - const prologues = await this.#evaluate_supplemental_chapters(this.story.prologues ?? []) - const chapters = await this.#evaluate_chapters(this.story.chapters) - const epilogues = await this.#evaluate_supplemental_chapters(this.story.epilogues ?? []) + const { evaluations: prologues, has_errors: prologue_errors } = await this.#evaluate_supplemental_chapters(story.prologues ?? [], dry_run) + const chapters = await this.#evaluate_chapters(story.chapters, prologue_errors, dry_run) + const { evaluations: epilogues } = await this.#evaluate_supplemental_chapters(story.epilogues ?? [], dry_run) return { - display_path: this.display_path, - full_path: this.full_path, - description: this.story.description, + display_path, + full_path, + description: story.description, chapters, prologues, epilogues, @@ -60,38 +52,38 @@ export default class StoryEvaluator { } } - async #evaluate_chapters (chapters: Chapter[]): Promise { + async #evaluate_chapters (chapters: Chapter[], has_errors: boolean, dry_run: boolean): Promise { const evaluations: ChapterEvaluation[] = [] for (const chapter of chapters) { - if (this.dry_run) { + if (dry_run) { const title = chapter.synopsis || `${chapter.method} ${chapter.path}` evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } }) } else { - const evaluator = new ChapterEvaluator(chapter) - const evaluation = await evaluator.evaluate(this.has_errors) - this.has_errors = this.has_errors || evaluation.overall.result === Result.ERROR + const evaluation = await this._chapter_evaluator.evaluate(chapter, has_errors) + has_errors = has_errors || evaluation.overall.result === Result.ERROR evaluations.push(evaluation) } } return evaluations } - async #evaluate_supplemental_chapters (chapters: SupplementalChapter[]): Promise { + async #evaluate_supplemental_chapters (chapters: SupplementalChapter[], dry_run: boolean): Promise<{ evaluations: ChapterEvaluation[], has_errors: boolean }> { + let has_errors = false const evaluations: ChapterEvaluation[] = [] for (const chapter of chapters) { const title = `${chapter.method} ${chapter.path}` - if (this.dry_run) { + if (dry_run) { evaluations.push({ title, overall: { result: Result.SKIPPED, message: 'Dry Run', error: undefined } }) } else { - const response = await this.chapter_reader.read(chapter) + const response = await this._chapter_reader.read(chapter) const status = chapter.status ?? [200, 201] if (status.includes(response.status)) evaluations.push({ title, overall: { result: Result.PASSED } }) else { - this.has_errors = true + has_errors = true evaluations.push({ title, overall: { result: Result.ERROR, message: response.message, error: response.error as Error } }) } } } - return evaluations + return { evaluations, has_errors } } } diff --git a/tools/src/tester/TestRunner.ts b/tools/src/tester/TestRunner.ts new file mode 100644 index 000000000..fa88306ab --- /dev/null +++ b/tools/src/tester/TestRunner.ts @@ -0,0 +1,66 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import type StoryEvaluator from './StoryEvaluator' +import { type StoryFile } from './StoryEvaluator' +import fs from 'fs' +import { type Story } from './types/story.types' +import { read_yaml } from '../../helpers' +import { Result, type StoryEvaluation } from './types/eval.types' +import { type ResultLogger } from './ResultLogger' +import { basename, resolve } from 'path' + +export default class TestRunner { + private readonly _story_evaluator: StoryEvaluator + private readonly _result_logger: ResultLogger + + constructor (story_evaluator: StoryEvaluator, result_logger: ResultLogger) { + this._story_evaluator = story_evaluator + this._result_logger = result_logger + } + + async run (story_path: string, dry_run: boolean = false): Promise<{ evaluations: StoryEvaluation[], failed: boolean }> { + let failed = false + const story_files = this.#sort_story_files(this.#collect_story_files(resolve(story_path), '', '')) + const evaluations: StoryEvaluation[] = [] + for (const story_file of story_files) { + const evaluation = await this._story_evaluator.evaluate(story_file, dry_run) + evaluations.push(evaluation) + this._result_logger.log(evaluation) + if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true + } + return { evaluations, failed } + } + + #collect_story_files (folder: string, file: string, prefix: string): StoryFile[] { + const path = file === '' ? folder : `${folder}/${file}` + const next_prefix = prefix === '' ? file : `${prefix}/${file}` + if (fs.statSync(path).isFile()) { + const story: Story = read_yaml(path) + return [{ + display_path: next_prefix === '' ? basename(path) : next_prefix, + full_path: path, + story + }] + } else { + return fs.readdirSync(path).flatMap(next_file => { + return this.#collect_story_files(path, next_file, next_prefix) + }) + } + } + + #sort_story_files (story_files: StoryFile[]): StoryFile[] { + return story_files.sort(({ display_path: a }, { display_path: b }) => { + const a_depth = a.split('/').length + const b_depth = b.split('/').length + if (a_depth !== b_depth) return a_depth - b_depth + return a.localeCompare(b) + }) + } +} diff --git a/tools/src/tester/TestsRunner.ts b/tools/src/tester/TestsRunner.ts deleted file mode 100644 index 94b00f543..000000000 --- a/tools/src/tester/TestsRunner.ts +++ /dev/null @@ -1,80 +0,0 @@ -/* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ - -import { type OpenAPIV3 } from 'openapi-types' -import SpecParser from './SpecParser' -import ChapterReader from './ChapterReader' -import SchemaValidator from './SchemaValidator' -import StoryEvaluator, { type StoryFile } from './StoryEvaluator' -import fs from 'fs' -import { type Story } from './types/story.types' -import { read_yaml } from '../../helpers' -import { Result, type StoryEvaluation } from './types/eval.types' -import ResultsDisplayer, { type TestRunOptions, type DisplayOptions } from './ResultsDisplayer' -import SharedResources from './SharedResources' -import { resolve, basename } from 'path' - -type TestsRunnerOptions = TestRunOptions & DisplayOptions & Record - -export default class TestsRunner { - path: string // Path to a story file or a directory containing story files - opts: TestsRunnerOptions - - constructor (spec: OpenAPIV3.Document, path: string, opts: TestsRunnerOptions) { - this.path = resolve(path) - this.opts = opts - - const chapter_reader = new ChapterReader() - const spec_parser = new SpecParser(spec) - const schema_validator = new SchemaValidator(spec) - SharedResources.create_instance({ chapter_reader, schema_validator, spec_parser }) - } - - async run (debug: boolean = false): Promise { - let failed = false - const story_files = this.#collect_story_files(this.path, '', '') - const evaluations: StoryEvaluation[] = [] - for (const story_file of this.#sort_story_files(story_files)) { - const evaluator = new StoryEvaluator(story_file, this.opts.dry_run) - const evaluation = await evaluator.evaluate() - const displayer = new ResultsDisplayer(evaluation, this.opts) - if (debug) evaluations.push(evaluation) - else displayer.display() - if ([Result.ERROR, Result.FAILED].includes(evaluation.result)) failed = true - } - if (failed && !debug) process.exit(1) - return evaluations - } - - #collect_story_files (folder: string, file: string, prefix: string): StoryFile[] { - const path = file === '' ? folder : `${folder}/${file}` - const next_prefix = prefix === '' ? file : `${prefix}/${file}` - if (fs.statSync(path).isFile()) { - const story: Story = read_yaml(path) - return [{ - display_path: next_prefix === '' ? basename(path) : next_prefix, - full_path: path, - story - }] - } else { - return fs.readdirSync(path).flatMap(next_file => { - return this.#collect_story_files(path, next_file, next_prefix) - }) - } - } - - #sort_story_files (story_files: StoryFile[]): StoryFile[] { - return story_files.sort((a, b) => { - const a_depth = a.display_path.split('/').length - const b_depth = b.display_path.split('/').length - if (a_depth !== b_depth) return a_depth - b_depth - return a.display_path.localeCompare(b.display_path) - }) - } -} diff --git a/tools/src/tester/start.ts b/tools/src/tester/start.ts index 797602158..41a9eb8ad 100644 --- a/tools/src/tester/start.ts +++ b/tools/src/tester/start.ts @@ -9,29 +9,55 @@ import OpenApiMerger from '../merger/OpenApiMerger' import { LogLevel } from '../Logger' -import TestsRunner from './TestsRunner' +import TestRunner from './TestRunner' import { Command, Option } from '@commander-js/extra-typings' -import _ from 'lodash' +import { + get_opensearch_opts_from_cli, + OPENSEARCH_INSECURE_OPTION, + OPENSEARCH_PASSWORD_OPTION, + OPENSEARCH_URL_OPTION, + OPENSEARCH_USERNAME_OPTION, OpenSearchHttpClient +} from '../OpenSearchHttpClient' +import ChapterReader from './ChapterReader' +import ChapterEvaluator from './ChapterEvaluator' +import OperationLocator from './OperationLocator' +import SchemaValidator from './SchemaValidator' +import StoryEvaluator from './StoryEvaluator' +import { ConsoleResultLogger } from './ResultLogger' +import * as process from 'node:process' const command = new Command() .description('Run test stories against the OpenSearch spec.') .addOption(new Option('--spec, --spec-path ', 'path to the root folder of the multi-file spec').default('./spec')) .addOption(new Option('--tests, --tests-path ', 'path to the root folder of the tests').default('./tests')) - .addOption(new Option('--tab-width ', 'tab width for displayed results').default('4')) - .addOption(new Option('--verbose', 'whether to print the full stack trace of errors')) - .addOption(new Option('--dry-run', 'dry run only, do not make HTTP requests')) + .addOption( + new Option('--tab-width ', 'tab width for displayed results') + .default(4) + .argParser((v, _) => Number.parseInt(v)) + ) + .addOption(new Option('--verbose', 'whether to print the full stack trace of errors').default(false)) + .addOption(new Option('--dry-run', 'dry run only, do not make HTTP requests').default(false)) + .addOption(OPENSEARCH_URL_OPTION) + .addOption(OPENSEARCH_USERNAME_OPTION) + .addOption(OPENSEARCH_PASSWORD_OPTION) + .addOption(OPENSEARCH_INSECURE_OPTION) .allowExcessArguments(false) .parse() const opts = command.opts() -const options = { - verbose: opts.verbose ?? false, - tab_width: Number.parseInt(opts.tabWidth), - dry_run: opts.dryRun ?? false -} -// The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml -process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' const spec = (new OpenApiMerger(opts.specPath, LogLevel.error)).merge() -const runner = new TestsRunner(spec, opts.testsPath, options) -void runner.run().then(() => { _.noop() }) + +const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli(opts)) +const chapter_reader = new ChapterReader(http_client) +const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec), chapter_reader, new SchemaValidator(spec)) +const story_evaluator = new StoryEvaluator(chapter_reader, chapter_evaluator) +const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose) +const runner = new TestRunner(story_evaluator, result_logger) + +runner.run(opts.testsPath, opts.dryRun) + .then( + ({ failed }) => { + if (failed) process.exit(1) + }, + err => { throw err }) diff --git a/tools/src/tester/types/eval.types.ts b/tools/src/tester/types/eval.types.ts index 34b0756eb..166d72c07 100644 --- a/tools/src/tester/types/eval.types.ts +++ b/tools/src/tester/types/eval.types.ts @@ -36,7 +36,7 @@ export interface ChapterEvaluation { export interface Evaluation { result: Result message?: string - error?: Error + error?: Error | string } export enum Result { diff --git a/tools/tests/tester/StoryEvaluator.test.ts b/tools/tests/tester/StoryEvaluator.test.ts index d1d26ff30..3edae05ea 100644 --- a/tools/tests/tester/StoryEvaluator.test.ts +++ b/tools/tests/tester/StoryEvaluator.test.ts @@ -7,45 +7,42 @@ * compatible open source license. */ -import { create_shared_resources, load_actual_evaluation, load_expected_evaluation } from './helpers' -import { read_yaml } from '../../helpers' -import { type OpenAPIV3 } from 'openapi-types' +import { construct_tester_components, load_actual_evaluation, load_expected_evaluation } from './helpers' -const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') -create_shared_resources(spec as OpenAPIV3.Document) +const { story_evaluator } = construct_tester_components('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') test('passed', async () => { - const actual = await load_actual_evaluation('passed') + const actual = await load_actual_evaluation(story_evaluator, 'passed') const expected = load_expected_evaluation('passed') expect(actual).toEqual(expected) }) test('skipped', async () => { - const actual = await load_actual_evaluation('skipped') + const actual = await load_actual_evaluation(story_evaluator, 'skipped') const expected = load_expected_evaluation('skipped') expect(actual).toEqual(expected) }) test('failed/not_found', async () => { - const actual = await load_actual_evaluation('failed/not_found') + const actual = await load_actual_evaluation(story_evaluator, 'failed/not_found') const expected = load_expected_evaluation('failed/not_found') expect(actual).toEqual(expected) }) test('failed/invalid_data', async () => { - const actual = await load_actual_evaluation('failed/invalid_data') + const actual = await load_actual_evaluation(story_evaluator, 'failed/invalid_data') const expected = load_expected_evaluation('failed/invalid_data') expect(actual).toEqual(expected) }) test('error/prologue_error', async () => { - const actual = await load_actual_evaluation('error/prologue_error') + const actual = await load_actual_evaluation(story_evaluator, 'error/prologue_error') const expected = load_expected_evaluation('error/prologue_error') expect(actual).toEqual(expected) }) test('error/chapter_error', async () => { - const actual = await load_actual_evaluation('error/chapter_error') + const actual = await load_actual_evaluation(story_evaluator, 'error/chapter_error') const expected = load_expected_evaluation('error/chapter_error') expect(actual).toEqual(expected) }) diff --git a/tools/tests/tester/TestRunner.test.ts b/tools/tests/tester/TestRunner.test.ts new file mode 100644 index 000000000..00f3c47c3 --- /dev/null +++ b/tools/tests/tester/TestRunner.test.ts @@ -0,0 +1,36 @@ +/* +* Copyright OpenSearch Contributors +* SPDX-License-Identifier: Apache-2.0 +* +* The OpenSearch Contributors require contributions made to +* this file be licensed under the Apache-2.0 license or a +* compatible open source license. +*/ + +import { construct_tester_components, flatten_errors, load_expected_evaluation } from './helpers' +import { type StoryEvaluation } from '../../src/tester/types/eval.types' + +test('stories folder', async () => { + const { test_runner } = construct_tester_components('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') + const result = await test_runner.run('tools/tests/tester/fixtures/stories') + + expect(result.failed).toBeTruthy() + + const actual_evaluations: Array> = [] + + for (const evaluation of result.evaluations) { + const { full_path, ...rest } = flatten_errors(evaluation) + expect(full_path.endsWith(rest.display_path)).toBeTruthy() + actual_evaluations.push(rest) + } + + const skipped = load_expected_evaluation('skipped', true) + const passed = load_expected_evaluation('passed', true) + const not_found = load_expected_evaluation('failed/not_found', true) + const invalid_data = load_expected_evaluation('failed/invalid_data', true) + const chapter_error = load_expected_evaluation('error/chapter_error', true) + const prologue_error = load_expected_evaluation('error/prologue_error', true) + + const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found] + expect(actual_evaluations).toEqual(expected_evaluations) +}) diff --git a/tools/tests/tester/TestsRunner.test.ts b/tools/tests/tester/TestsRunner.test.ts deleted file mode 100644 index c3f9e94a6..000000000 --- a/tools/tests/tester/TestsRunner.test.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* -* Copyright OpenSearch Contributors -* SPDX-License-Identifier: Apache-2.0 -* -* The OpenSearch Contributors require contributions made to -* this file be licensed under the Apache-2.0 license or a -* compatible open source license. -*/ - -import { read_yaml } from '../../helpers' -import TestsRunner from '../../src/tester/TestsRunner' -import { type OpenAPIV3 } from 'openapi-types' -import { load_expected_evaluation, scrub_errors } from './helpers' - -test('stories folder', async () => { - // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml - process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' - const spec = read_yaml('tools/tests/tester/fixtures/specs/indices_excerpt.yaml') - const runner = new TestsRunner(spec as OpenAPIV3.Document, 'tools/tests/tester/fixtures/stories', {}) - const actual_evaluations = await runner.run(true) as any[] - for (const evaluation of actual_evaluations) scrub_errors(evaluation) - for (const evaluation of actual_evaluations) { - expect(evaluation.full_path.endsWith(evaluation.display_path)).toBeTruthy() - delete evaluation.full_path - } - - const skipped = load_expected_evaluation('skipped', true) - const passed = load_expected_evaluation('passed', true) - const not_found = load_expected_evaluation('failed/not_found', true) - const invalid_data = load_expected_evaluation('failed/invalid_data', true) - const chapter_error = load_expected_evaluation('error/chapter_error', true) - const prologue_error = load_expected_evaluation('error/prologue_error', true) - - const expected_evaluations = [passed, skipped, chapter_error, prologue_error, invalid_data, not_found] - expect(actual_evaluations).toEqual(expected_evaluations) -}) diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 86782c818..e10accce6 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -7,49 +7,109 @@ * compatible open source license. */ -import ChapterReader from '../../src/tester/ChapterReader' -import SpecParser from '../../src/tester/SpecParser' -import SchemaValidator from '../../src/tester/SchemaValidator' -import SharedResources from '../../src/tester/SharedResources' -import { type OpenAPIV3 } from 'openapi-types' import YAML from 'yaml' -import type { StoryEvaluation } from '../../src/tester/types/eval.types' -import type { Story } from '../../src/tester/types/story.types' +import type { ChapterEvaluation, Evaluation, StoryEvaluation } from '../../src/tester/types/eval.types' import { read_yaml } from '../../helpers' import StoryEvaluator from '../../src/tester/StoryEvaluator' +import OperationLocator from '../../src/tester/OperationLocator' +import ChapterReader from '../../src/tester/ChapterReader' +import SchemaValidator from '../../src/tester/SchemaValidator' +import ChapterEvaluator from '../../src/tester/ChapterEvaluator' +import { OpenSearchHttpClient } from '../../src/OpenSearchHttpClient' +import { type OpenAPIV3 } from 'openapi-types' +import TestRunner from '../../src/tester/TestRunner' +import { NoOpResultLogger, type ResultLogger } from '../../src/tester/ResultLogger' +import * as process from 'node:process' -export function create_shared_resources (spec: any): void { - // The fallback password must match the default password specified in .github/opensearch-cluster/docker-compose.yml - process.env.OPENSEARCH_PASSWORD = process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' - const chapter_reader = new ChapterReader() - const spec_parser = new SpecParser(spec as OpenAPIV3.Document) - const schema_validator = new SchemaValidator(spec as OpenAPIV3.Document) - SharedResources.create_instance({ chapter_reader, schema_validator, spec_parser }) +export function construct_tester_components (spec_path: string): { + specification: OpenAPIV3.Document + operation_locator: OperationLocator + opensearch_http_client: OpenSearchHttpClient + chapter_reader: ChapterReader + schema_validator: SchemaValidator + chapter_evaluator: ChapterEvaluator + story_evaluator: StoryEvaluator + result_logger: ResultLogger + test_runner: TestRunner +} { + const specification: OpenAPIV3.Document = read_yaml(spec_path) + const operation_locator = new OperationLocator(specification) + const opensearch_http_client = new OpenSearchHttpClient({ + insecure: true, + username: process.env.OPENSEARCH_USERNAME ?? 'admin', + password: process.env.OPENSEARCH_PASSWORD ?? 'myStrongPassword123!' + }) + const chapter_reader = new ChapterReader(opensearch_http_client) + const schema_validator = new SchemaValidator(specification) + const chapter_evaluator = new ChapterEvaluator(operation_locator, chapter_reader, schema_validator) + const story_evaluator = new StoryEvaluator(chapter_reader, chapter_evaluator) + const result_logger = new NoOpResultLogger() + const test_runner = new TestRunner(story_evaluator, result_logger) + return { + specification, + operation_locator, + opensearch_http_client, + chapter_reader, + schema_validator, + chapter_evaluator, + story_evaluator, + result_logger, + test_runner + } } export function print_yaml (obj: any): void { console.log(YAML.stringify(obj, { indent: 2, singleQuote: true, lineWidth: undefined })) } -export function scrub_errors (obj: any): void { - for (const key in obj) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - if (key === 'error') obj[key] = obj[key].message - else if (typeof obj[key] === 'object') scrub_errors(obj[key]) +export function flatten_errors (evaluation: StoryEvaluation): StoryEvaluation { + const flatten = (e: T): T => (e !== undefined + ? { + ...e, + error: typeof e.error === 'object' ? e.error.message : e.error + } + : undefined as T) + + const flatten_chapters = (chapters: T): T => { + if (chapters === undefined) return undefined as T + return chapters.map((c: ChapterEvaluation): ChapterEvaluation => ({ + ...c, + overall: flatten(c.overall), + request: c.request !== undefined + ? { + parameters: c.request.parameters !== undefined + ? Object.fromEntries(Object.entries(c.request.parameters).map(([k, v]) => [k, flatten(v)])) + : undefined, + request_body: flatten(c.request.request_body) + } + : undefined, + response: c.response !== undefined + ? { + status: flatten(c.response.status), + payload: flatten(c.response.payload) + } + : undefined + })) as T + } + + return { + ...evaluation, + chapters: flatten_chapters(evaluation.chapters), + epilogues: flatten_chapters(evaluation.epilogues), + prologues: flatten_chapters(evaluation.prologues) } } -export function load_expected_evaluation (name: string, exclude_full_path: boolean = false): Record { - const expected = read_yaml(`tools/tests/tester/fixtures/evals/${name}.yaml`) - if (exclude_full_path) delete expected.full_path - return expected +export function load_expected_evaluation (name: string, exclude_full_path: boolean = false): Omit & { full_path?: string } { + const { full_path, ...rest }: StoryEvaluation = read_yaml(`tools/tests/tester/fixtures/evals/${name}.yaml`) + return !exclude_full_path ? { ...rest, full_path } : rest } -export async function load_actual_evaluation (name: string): Promise { - const story: Story = read_yaml(`tools/tests/tester/fixtures/stories/${name}.yaml`) - const display_path = `${name}.yaml` +export async function load_actual_evaluation (evaluator: StoryEvaluator, name: string): Promise { const full_path = `tools/tests/tester/fixtures/stories/${name}.yaml` - const actual = await new StoryEvaluator({ display_path, full_path, story }).evaluate() - scrub_errors(actual) - return actual + return flatten_errors(await evaluator.evaluate({ + full_path, + display_path: `${name}.yaml`, + story: read_yaml(full_path) + })) }