diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index 59a9b2f5c..e4821d86d 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -80,11 +80,20 @@ export default class ChapterEvaluator { #evaluate_status(chapter: Chapter, response: ActualResponse): Evaluation { const expected_status = chapter.response?.status ?? 200 if (response.status === expected_status) return { result: Result.PASSED } - return { + + var result: Evaluation = { result: Result.ERROR, - message: `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}. ${response.message}`, - error: response.error as Error + message: _.join(_.compact([ + `Expected status ${expected_status}, but received ${response.status}: ${response.content_type}.`, + response.message + ]), ' ') } + + if (response.error !== undefined) { + result.error = response.error as Error + } + + return result } #evaluate_payload_body(response: ActualResponse, expected_payload?: Payload): Evaluation { diff --git a/tools/src/tester/SchemaValidator.ts b/tools/src/tester/SchemaValidator.ts index 9fdc0dfb6..eddf54a09 100644 --- a/tools/src/tester/SchemaValidator.ts +++ b/tools/src/tester/SchemaValidator.ts @@ -45,9 +45,15 @@ export default class SchemaValidator { this.logger.info(`* ${to_json(data)}`) this.logger.info(`& ${to_json(validate.errors)}`) } - return { + + var result: Evaluation = { result: valid ? Result.PASSED : Result.FAILED, - message: valid ? undefined : this.ajv.errorsText(validate.errors) } + + if (!valid) { + result.message = this.ajv.errorsText(validate.errors) + } + + return result } } diff --git a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml index 669fefd08..704c71ba4 100644 --- a/tools/tests/tester/fixtures/evals/error/chapter_error.yaml +++ b/tools/tests/tester/fixtures/evals/error/chapter_error.yaml @@ -12,8 +12,8 @@ prologues: chapters: - title: This chapter should fail. overall: - result: FAILED message: Operation "GET /{index}/settings" not found in the spec. + result: FAILED - title: This chapter show throw an error. overall: result: ERROR diff --git a/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml b/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml index 3bf4abbac..65a65f681 100644 --- a/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml +++ b/tools/tests/tester/fixtures/evals/failed/invalid_data.yaml @@ -1,7 +1,7 @@ display_path: failed/invalid_data.yaml full_path: tools/tests/tester/fixtures/stories/failed/invalid_data.yaml -result: FAILED +result: ERROR description: This story should failed due invalid data. prologues: [] @@ -59,6 +59,23 @@ chapters: payload_schema: result: FAILED message: data must NOT have additional properties + - title: This chapter should fail because the response status does not match. + overall: + result: ERROR + request: + parameters: + index: + result: PASSED + request_body: + result: PASSED + response: + status: + result: ERROR + message: 'Expected status 404, but received 200: application/json.' + payload_body: + result: SKIPPED + payload_schema: + result: SKIPPED epilogues: - title: DELETE /books diff --git a/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml b/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml index edac0cdd7..486cdfb90 100644 --- a/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml +++ b/tools/tests/tester/fixtures/stories/failed/invalid_data.yaml @@ -32,3 +32,10 @@ chapters: payload: acknowledged: false shards_acknowledged: true + - synopsis: This chapter should fail because the response status does not match. + path: /{index} + method: PUT + parameters: + index: books + response: + status: 404 diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 482e2a6c9..748c21e3d 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -72,34 +72,51 @@ export function print_yaml (obj: any): void { } export function flatten_errors (evaluation: StoryEvaluation): StoryEvaluation { - const flatten = (e: T): T => (e !== undefined - ? { - ...e, - error: typeof e.error === 'object' ? e.error.message : e.error + const flatten = (e: T): T => { + + var result = e + + if (e !== undefined && result !== undefined) { + if (typeof e.error === 'object' && e.error.message !== undefined) { + result.error = e.error.message + } else if (e.error !== undefined) { + result.error = e.error + } } - : undefined as T) + + return result + } const flatten_chapters = (chapters: T): T => { if (chapters === undefined) return undefined as T - return chapters.map((c: ChapterEvaluation): ChapterEvaluation => ({ - ...c, - overall: flatten(c.overall), - request: c.request !== undefined - ? { - parameters: c.request.parameters !== undefined - ? Object.fromEntries(Object.entries(c.request.parameters).map(([k, v]) => [k, flatten(v)])) - : undefined, + return chapters.map((c: ChapterEvaluation): ChapterEvaluation => { + var result = { + ...c, + overall: flatten(c.overall), + } + + if (c.request !== undefined) { + result.request = { request_body: flatten(c.request.request_body) } - : undefined, - response: c.response !== undefined - ? { + + if (c.request.parameters !== undefined) { + result.request.parameters = Object.fromEntries( + Object.entries(c.request.parameters).map(([k, v]) => [k, flatten(v)]) + ) + } + } + + if (c.response !== undefined) { + result.response = { status: flatten(c.response.status), payload_body: flatten(c.response.payload_body), payload_schema: flatten(c.response.payload_schema) } - : undefined - })) as T + } + + return result; + }) as T } return {