Skip to content

Commit

Permalink
Allow response payload to reference variables. (#471)
Browse files Browse the repository at this point in the history
Signed-off-by: dblock <[email protected]>
  • Loading branch information
dblock authored Aug 6, 2024
1 parent c7dbaaf commit 870a018
Show file tree
Hide file tree
Showing 12 changed files with 218 additions and 66 deletions.
5 changes: 3 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Added `concurrent_query_*` and `search_idle_reactivate_count_total` fields to `SearchStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395))
- Added `remote_store` to `TranslogStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395))
- Added `file` to `/_cache/clear` and `/{index}/_cache/clear` ([#396](https://github.com/opensearch-project/opensearch-api-specification/pull/396))
- Add `strict_allow_templates` option for the dynamic mapping parameter ([#408](https://github.com/opensearch-project/opensearch-api-specification/pull/408))
- Added `strict_allow_templates` option for the dynamic mapping parameter ([#408](https://github.com/opensearch-project/opensearch-api-specification/pull/408))
- Added a workflow to run tests against the next version of OpenSearch ([#409](https://github.com/opensearch-project/opensearch-api-specification/pull/409))
- Added support for skipping tests using semver range ([#410](https://github.com/opensearch-project/opensearch-api-specification/pull/410))
- Added `cluster_manager_timeout` to `HEAD /{index}` ([#421](https://github.com/opensearch-project/opensearch-api-specification/pull/421))
Expand All @@ -65,7 +65,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
- Added metadata additionalProperties to `ErrorCause` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462))
- Added `creation_date` field to `DanglingIndex` ([#462](https://github.com/opensearch-project/opensearch-api-specification/pull/462))
- Added doc on `cluster create-index blocked` workaround ([#465](https://github.com/opensearch-project/opensearch-api-specification/pull/465))

- Added support for reusing output variables as keys in payload expectations ([#471](https://github.com/opensearch-project/opensearch-api-specification/pull/471))

### Changed

- Replaced Smithy with a native OpenAPI spec ([#189](https://github.com/opensearch-project/opensearch-api-specification/issues/189))
Expand Down
2 changes: 2 additions & 0 deletions TESTING_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,8 @@ Consider the following chapters in [ml/model_groups](tests/ml/model_groups.yaml)
```
As you can see, the `output` field in the first chapter saves the `model_group_id` from the response body. This value is then used in the subsequent chapters to query and delete the model group.
You can also reuse output in payload expectations. See [tests/nodes/plugins/index_state_management.yaml](tests/nodes/plugins/index_state_management.yaml) for an example.
### Managing Versions
It's common to add a feature to the next version of OpenSearch. When adding a new API in the spec, make sure to specify `x-version-added`, `x-version-deprecated` or `x-version-removed`. Finally, specify a semver range in your test stories or chapters as follows.
Expand Down
30 changes: 30 additions & 0 deletions tests/nodes/plugins/index_state_management.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
$schema: ../../../json_schemas/test_story.schema.yaml

description: Get index_state_management node info settings.
prologues:
- path: /_cat/nodes
id: node
method: GET
parameters:
full_id: true
size: 1
format: json
h: id
output:
id: payload[0].id
chapters:
- synopsis: Get node info.
path: /_nodes/{node_id_or_metric}
method: GET
parameters:
node_id_or_metric: ${node.id}
response:
status: 200
payload:
nodes:
${node.id}:
settings:
plugins:
index_state_management:
job_interval: '1'

5 changes: 4 additions & 1 deletion tools/src/tester/ChapterEvaluator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,12 @@ export default class ChapterEvaluator {
const params = this.#evaluate_parameters(chapter, operation)
const request = this.#evaluate_request(chapter, operation)
const status = this.#evaluate_status(chapter, response)
const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, chapter.response?.payload) : { result: Result.SKIPPED }
const payload_schema_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_schema(chapter, response, operation) : { result: Result.SKIPPED }
const output_values_evaluation: EvaluationWithOutput = status.result === Result.PASSED ? ChapterOutput.extract_output_values(response, chapter.output) : { evaluation: { result: Result.SKIPPED } }
const response_payload: Payload | undefined = status.result === Result.PASSED ? story_outputs.resolve_value(chapter.response?.payload) : chapter.response?.payload
const payload_body_evaluation = status.result === Result.PASSED ? this.#evaluate_payload_body(response, response_payload) : { result: Result.SKIPPED }

if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`)

const evaluations = _.compact(_.concat(
Object.values(params),
Expand Down
8 changes: 3 additions & 5 deletions tools/src/tester/ChapterOutput.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,19 +30,17 @@ export class ChapterOutput {
if (!output) return { evaluation: { result: Result.SKIPPED } }
const chapter_output = new ChapterOutput({})
for (const [name, path] of Object.entries(output)) {
const [source, ...rest] = path.split('.')
const keys = rest.join('.')
let value: any
if (source === 'payload') {
if (path == 'payload' || path.startsWith('payload.') || path.match(/^payload\[\d*\]/)) {
if (response.payload === undefined) {
return { evaluation: { result: Result.ERROR, message: 'No payload found in response, but expected output: ' + path } }
}
value = keys.length === 0 ? response.payload : _.get(response.payload, keys)
value = _.get(response, path)
if (value === undefined) {
return { evaluation: { result: Result.ERROR, message: `Expected to find non undefined value at \`${path}\`.` } }
}
} else {
return { evaluation: { result: Result.ERROR, message: 'Unknown output source: ' + source } }
return { evaluation: { result: Result.ERROR, message: `Unknown output source: ${path.split('.')[0]}.` } }
}
chapter_output.set(name, value)
}
Expand Down
6 changes: 4 additions & 2 deletions tools/src/tester/StoryOutputs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,14 @@ export class StoryOutputs {
resolved_array.push(this.resolve_value(value))
}
return resolved_array
} else {
} else if (payload !== null) {
const resolved_obj: Record<string, any> = {}
for (const [key, value] of Object.entries(payload as Record<string, any>)) {
resolved_obj[key] = this.resolve_value(value)
resolved_obj[this.resolve_value(key)] = this.resolve_value(value)
}
return resolved_obj
} else {
return payload
}
default:
return payload
Expand Down
7 changes: 6 additions & 1 deletion tools/src/tester/SupplementalChapterEvaluator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,24 @@ import { StoryOutputs } from "./StoryOutputs";
import { overall_result } from "./helpers";
import { ChapterEvaluation, Result } from "./types/eval.types";
import { SupplementalChapter } from "./types/story.types";
import { Logger } from "../Logger";
import { to_json } from "../helpers";

export default class SupplementalChapterEvaluator {
private readonly _chapter_reader: ChapterReader;
private readonly logger: Logger;

constructor(chapter_reader: ChapterReader) {
constructor(chapter_reader: ChapterReader, logger: Logger) {
this._chapter_reader = chapter_reader;
this.logger = logger
}

async evaluate(chapter: SupplementalChapter, story_outputs: StoryOutputs): Promise<{ evaluation: ChapterEvaluation, evaluation_error: boolean }> {
const title = `${chapter.method} ${chapter.path}`
const response = await this._chapter_reader.read(chapter, story_outputs)
const status = chapter.status ?? [200, 201]
const output_values_evaluation = ChapterOutput.extract_output_values(response, chapter.output)
if (output_values_evaluation.output) this.logger.info(`$ ${to_json(output_values_evaluation.output)}`)
let response_evaluation: ChapterEvaluation
const passed_evaluation = { title, overall: { result: Result.PASSED } }
if (status.includes(response.status)) {
Expand Down
2 changes: 1 addition & 1 deletion tools/src/tester/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ const spec = new MergedOpenApiSpec(opts.specPath, opts.opensearchVersion, new Lo
const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli({ opensearchResponseType: 'arraybuffer', ...opts }))
const chapter_reader = new ChapterReader(http_client, logger)
const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec.spec()), chapter_reader, new SchemaValidator(spec.spec(), logger), logger)
const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader)
const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader, logger)
const story_validator = new StoryValidator()
const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator)
const result_logger = new ConsoleResultLogger(opts.tabWidth, opts.verbose)
Expand Down
158 changes: 158 additions & 0 deletions tools/tests/tester/ChapterOutput.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

import { ChapterOutput } from 'tester/ChapterOutput'
import { EvaluationWithOutput, Result } from 'tester/types/eval.types'
import { ActualResponse } from 'tester/types/story.types'

function create_response(payload: any): ActualResponse {
return {
status: 200,
content_type: 'application/json',
payload
}
}

function passed_output(output: Record<string, any>): EvaluationWithOutput {
return {
evaluation: { result: Result.PASSED },
output: new ChapterOutput(output)
}
}

describe('with an object response', () => {
const response: ActualResponse = create_response({
a: {
b: {
c: 1
},
arr: [
{ d: 2 },
{ e: 3 }
]
}
})

test('returns nested values', () => {
const output = {
c: 'payload.a.b.c',
d: 'payload.a.arr[0].d',
e: 'payload.a.arr[1].e'
}

expect(ChapterOutput.extract_output_values(response, output)).toEqual(passed_output({
c: 1,
d: 2,
e: 3
}))
})

test('extracts complete payload', () => {
expect(ChapterOutput.extract_output_values(response, { x: 'payload' })).toEqual(
passed_output({ x: response.payload })
)
})

test('errors on undefined value', () => {
expect(ChapterOutput.extract_output_values(response, { x: 'payload.a.b.x[0]' })).toEqual({
evaluation: {
result: Result.ERROR,
message: 'Expected to find non undefined value at `payload.a.b.x[0]`.'
}
})
})

test('errors on invalid source', () => {
expect(ChapterOutput.extract_output_values(response, { x: 'foobar' })).toEqual({
evaluation: {
result: Result.ERROR,
message: 'Unknown output source: foobar.'
}
})
})
})

describe('with an array response', () => {
const response: ActualResponse = create_response([
{
a: {
b: {
c: 1
},
arr: [
{ d: 2 },
{ e: 3 }
]
}
},{
a: {
b: {
c: 2
},
arr: [
{ d: 3 },
{ e: 4 }
]
}
},
])

test('returns nested values', () => {
const output = {
c1: 'payload[0].a.b.c',
d1: 'payload[0].a.arr[0].d',
e1: 'payload[0].a.arr[1].e',
c2: 'payload[1].a.b.c',
d2: 'payload[1].a.arr[0].d',
e2: 'payload[1].a.arr[1].e'
}

expect(ChapterOutput.extract_output_values(response, output)).toEqual(passed_output({
c1: 1,
d1: 2,
e1: 3,
c2: 2,
d2: 3,
e2: 4
}))
})

test('extracts complete payload', () => {
expect(ChapterOutput.extract_output_values(response, { x: 'payload' })).toEqual(
passed_output({ x: response.payload })
)
})

test('errors on undefined value', () => {
expect(ChapterOutput.extract_output_values(response, { x: 'payload[0].a.b.x[0]' })).toEqual({
evaluation: {
result: Result.ERROR,
message: 'Expected to find non undefined value at `payload[0].a.b.x[0]`.'
}
})
})

test('errors on invalid source', () => {
expect(ChapterOutput.extract_output_values(response, { x: 'foobar' })).toEqual({
evaluation: {
result: Result.ERROR,
message: 'Unknown output source: foobar.'
}
})
})

test('errors on invalid index', () => {
expect(ChapterOutput.extract_output_values(response, { x: 'payload[2]' })).toEqual({
evaluation: {
result: Result.ERROR,
message: 'Expected to find non undefined value at `payload[2]`.'
}
})
})
})
6 changes: 4 additions & 2 deletions tools/tests/tester/StoryOutputs.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ test('resolve_value', () => {
e: 'str',
f: true
},
g: 123
g: 123,
'${chapter_id.x}': 345
}
expect(story_outputs.resolve_value(value)).toEqual(
{
Expand All @@ -43,7 +44,8 @@ test('resolve_value', () => {
e: 'str',
f: true
},
g: 123
g: 123,
1: 345
}
)
})
Expand Down
2 changes: 1 addition & 1 deletion tools/tests/tester/helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ export function construct_tester_components (spec_path: string): {
const chapter_reader = new ChapterReader(opensearch_http_client, logger)
const schema_validator = new SchemaValidator(specification, logger)
const chapter_evaluator = new ChapterEvaluator(operation_locator, chapter_reader, schema_validator, logger)
const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader)
const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader, logger)
const story_validator = new StoryValidator()
const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator)
const result_logger = new NoOpResultLogger()
Expand Down
Loading

0 comments on commit 870a018

Please sign in to comment.