From 9bace5577f9006cfa5881cff46bc13308f2a035d Mon Sep 17 00:00:00 2001 From: dblock Date: Mon, 1 Jul 2024 19:01:39 -0400 Subject: [PATCH] Added CBOR support. Signed-off-by: dblock --- CHANGELOG.md | 3 +- package-lock.json | 29 +++++++++++++++++++ package.json | 2 ++ spec/namespaces/cat.yaml | 10 +++++++ tests/cat/health.yaml | 13 +++++++++ tests/cat/indices.yaml | 8 +++++ tools/src/OpenSearchHttpClient.ts | 3 +- .../dump-cluster-spec/dump-cluster-spec.ts | 2 +- tools/src/tester/ChapterEvaluator.ts | 16 ++++++++-- tools/src/tester/ChapterReader.ts | 18 +++++++++--- tools/src/tester/test.ts | 2 +- tools/tests/tester/fixtures/evals/passed.yaml | 16 ++++++++++ .../tests/tester/fixtures/specs/excerpt.yaml | 5 ++++ .../tests/tester/fixtures/stories/passed.yaml | 10 +++++++ tools/tests/tester/helpers.ts | 2 +- 15 files changed, 127 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0077fb523..1951741db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added support for `application/yaml` responses ([#363](https://github.com/opensearch-project/opensearch-api-specification/pull/363)) - Added test for search with seq_no_primary_term ([#367](https://github.com/opensearch-project/opensearch-api-specification/pull/367)) - Added a linter for parameter sorting ([#369](https://github.com/opensearch-project/opensearch-api-specification/pull/369)) - +- Added support for `application/cbor` responses ([#371](https://github.com/opensearch-project/opensearch-api-specification/pull/371)) + ### Changed - Replaced Smithy with a native OpenAPI spec ([#189](https://github.com/opensearch-project/opensearch-api-specification/issues/189)) diff --git a/package-lock.json b/package-lock.json index ed51006b5..e36bdf49b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,10 +27,12 @@ "@eslint/eslintrc": "^3.0.2", "@eslint/js": "^9.1.1", "@stylistic/eslint-plugin": "^2.3.0", + "@types/cbor-js": "^0.1.1", "@types/jest": "^29.5.12", "@types/qs": "^6.9.15", "@typescript-eslint/eslint-plugin": "^6.21.0", "ajv-errors": "^3.0.0", + "cbor": "^9.0.2", "eslint": "^8.57.0", "eslint-config-standard-with-typescript": "^43.0.1", "eslint-plugin-eslint-comments": "^3.2.0", @@ -2321,6 +2323,12 @@ "@babel/types": "^7.20.7" } }, + "node_modules/@types/cbor-js": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@types/cbor-js/-/cbor-js-0.1.1.tgz", + "integrity": "sha512-pfCx/EZC7VNBThwAQ0XvGPOXYm8BUk+gSVonaIGcEKBuqGJHTdcwAGW8WZkdRs/u9n9yOt1pBoPTCS1s8ZYpEQ==", + "dev": true + }, "node_modules/@types/eslint": { "version": "8.56.10", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.56.10.tgz", @@ -3255,6 +3263,18 @@ } ] }, + "node_modules/cbor": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/cbor/-/cbor-9.0.2.tgz", + "integrity": "sha512-JPypkxsB10s9QOWwa6zwPzqE1Md3vqpPc+cai4sAecuCsRyAtAl/pMyhPlMbT/xtPnm2dznJZYRLui57qiRhaQ==", + "dev": true, + "dependencies": { + "nofilter": "^3.1.0" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -6955,6 +6975,15 @@ "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==", "dev": true }, + "node_modules/nofilter": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/nofilter/-/nofilter-3.1.0.tgz", + "integrity": "sha512-l2NNj07e9afPnhAhvgVrCD/oy2Ai1yfLpuo3EpiO1jFTsB4sFz6oIfAfSZyQzVpkZQ9xS8ZS5g1jCBgq4Hwo0g==", + "dev": true, + "engines": { + "node": ">=12.19" + } + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", diff --git a/package.json b/package.json index cd974bb66..ab58e9cf6 100644 --- a/package.json +++ b/package.json @@ -37,10 +37,12 @@ "@eslint/eslintrc": "^3.0.2", "@eslint/js": "^9.1.1", "@stylistic/eslint-plugin": "^2.3.0", + "@types/cbor-js": "^0.1.1", "@types/jest": "^29.5.12", "@types/qs": "^6.9.15", "@typescript-eslint/eslint-plugin": "^6.21.0", "ajv-errors": "^3.0.0", + "cbor": "^9.0.2", "eslint": "^8.57.0", "eslint-config-standard-with-typescript": "^43.0.1", "eslint-plugin-eslint-comments": "^3.2.0", diff --git a/spec/namespaces/cat.yaml b/spec/namespaces/cat.yaml index 4f12c0e7e..e5da6dd6b 100644 --- a/spec/namespaces/cat.yaml +++ b/spec/namespaces/cat.yaml @@ -853,6 +853,11 @@ components: type: array items: $ref: '../schemas/cat.health.yaml#/components/schemas/HealthRecord' + application/cbor: + schema: + type: array + items: + $ref: '../schemas/cat.health.yaml#/components/schemas/HealthRecord' cat.help@200: description: '' content: @@ -874,6 +879,11 @@ components: type: array items: $ref: '../schemas/cat.indices.yaml#/components/schemas/IndicesRecord' + application/cbor: + schema: + type: array + items: + $ref: '../schemas/cat.indices.yaml#/components/schemas/IndicesRecord' cat.master@200: description: '' content: diff --git a/tests/cat/health.yaml b/tests/cat/health.yaml index 26184eb82..15ac180e5 100644 --- a/tests/cat/health.yaml +++ b/tests/cat/health.yaml @@ -69,3 +69,16 @@ chapters: status: yellow node.data: '1' discovered_cluster_manager: 'true' + - synopsis: Cat in different formats (format=cbor). + method: GET + path: /_cat/health + parameters: + format: cbor + response: + status: 200 + content_type: application/yaml + payload: + - node.total: '1' + status: yellow + node.data: '1' + discovered_cluster_manager: 'true' diff --git a/tests/cat/indices.yaml b/tests/cat/indices.yaml index ad5072424..d6a9b5a91 100644 --- a/tests/cat/indices.yaml +++ b/tests/cat/indices.yaml @@ -70,3 +70,11 @@ chapters: response: status: 200 content_type: application/yaml + - synopsis: Cat in different formats (format=cbor). + method: GET + path: /_cat/indices + parameters: + format: cbor + response: + status: 200 + content_type: application/cbor diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index 04d636f0c..90584cc82 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -79,7 +79,8 @@ export class OpenSearchHttpClient { password: opts.password } : undefined, - httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }) + httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }), + responseType: 'arraybuffer', }) } diff --git a/tools/src/dump-cluster-spec/dump-cluster-spec.ts b/tools/src/dump-cluster-spec/dump-cluster-spec.ts index cedc7094a..ca6fed892 100644 --- a/tools/src/dump-cluster-spec/dump-cluster-spec.ts +++ b/tools/src/dump-cluster-spec/dump-cluster-spec.ts @@ -33,7 +33,7 @@ async function main (opts: CommandOpts): Promise { const cluster_spec = await client.get('/_plugins/api') - write_yaml(opts.output, cluster_spec.data) + write_yaml(opts.output, Buffer.from(cluster_spec.data as string, 'binary').toString()) } const command = new Command() diff --git a/tools/src/tester/ChapterEvaluator.ts b/tools/src/tester/ChapterEvaluator.ts index 28ba11917..2357e4da5 100644 --- a/tools/src/tester/ChapterEvaluator.ts +++ b/tools/src/tester/ChapterEvaluator.ts @@ -19,16 +19,21 @@ import { ChapterOutput } from './ChapterOutput' import { Operation, atomizeChangeset, diff } from 'json-diff-ts' import YAML from 'yaml' import _ from 'lodash' +import CBOR from 'cbor' +import { Logger } from 'Logger' +import { to_json } from '../helpers' export default class ChapterEvaluator { + private readonly logger: Logger private readonly _operation_locator: OperationLocator private readonly _chapter_reader: ChapterReader private readonly _schema_validator: SchemaValidator - constructor(spec_parser: OperationLocator, chapter_reader: ChapterReader, schema_validator: SchemaValidator) { + constructor(spec_parser: OperationLocator, chapter_reader: ChapterReader, schema_validator: SchemaValidator, logger: Logger) { this._operation_locator = spec_parser this._chapter_reader = chapter_reader this._schema_validator = schema_validator + this.logger = logger } async evaluate(chapter: Chapter, skip: boolean, story_outputs: StoryOutputs): Promise { @@ -88,6 +93,7 @@ export default class ChapterEvaluator { if (expected_payload == null) return { result: Result.PASSED } const content_type = response.content_type ?? 'application/json' const payload = this.#deserialize_payload(response.payload, content_type) + this.logger.info(`${to_json(payload)}`) const delta = atomizeChangeset(diff(expected_payload, payload)) const messages: string[] = _.compact(delta.map((value, _index, _array) => { switch (value.type) { @@ -111,9 +117,13 @@ export default class ChapterEvaluator { #deserialize_payload(payload: any, content_type: string): any { if (payload === undefined) return undefined + const payload_data = payload as string switch (content_type) { - case 'application/yaml': return YAML.parse(payload as string) - default: return payload + case 'text/plain': return Buffer.from(payload_data, 'binary').toString() + case 'application/json': return JSON.parse(Buffer.from(payload_data, 'binary').toString()) + case 'application/yaml': return YAML.parse(Buffer.from(payload_data, 'binary').toString()) + case 'application/cbor': return CBOR.decode(payload_data) + default: return Buffer.from(payload_data, 'binary').toString() } } } diff --git a/tools/src/tester/ChapterReader.ts b/tools/src/tester/ChapterReader.ts index 9ea0a1e95..2e9db0e96 100644 --- a/tools/src/tester/ChapterReader.ts +++ b/tools/src/tester/ChapterReader.ts @@ -43,7 +43,7 @@ export default class ChapterReader { return qs.stringify(params, { arrayFormat: 'comma' }) } }).then(r => { - this.logger.info(`<= ${r.status} (${r.headers['content-type']}) | ${to_json(r.data)}`) + this.logger.info(`<= ${r.status} (${r.headers['content-type']}) | ${r.data?.length ?? 0} byte(s)`) response.status = r.status response.content_type = r.headers['content-type'].split(';')[0] response.payload = r.data @@ -54,11 +54,21 @@ export default class ChapterReader { } response.status = e.response.status response.content_type = e.response.headers['content-type'].split(';')[0] - response.payload = e.response.data?.error - response.message = e.response.data?.error?.reason ?? e.response.statusText + + try { + if (e.response.data !== undefined) { + const data = Buffer.from(e.response.data as string, 'binary').toString() + const payload = JSON.parse(data) + response.payload = payload?.error + response.message = payload.error?.reason ?? e.response.statusText + } + } catch { + // ignore + } + response.error = e - this.logger.info(`<= ${response.status} (${response.content_type}) | ${to_json(response.payload ?? response.message)}`) + this.logger.info(`<= ${response.status} (${response.content_type}) | ${(response.payload ?? response.message ?? '').length} byte(s)`) }) return response as ActualResponse } diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index 6c84c2b30..726e1f2fb 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -53,7 +53,7 @@ const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn) const spec = (new MergedOpenApiSpec(opts.specPath, new Logger(LogLevel.error))).spec() const http_client = new OpenSearchHttpClient(get_opensearch_opts_from_cli(opts)) const chapter_reader = new ChapterReader(http_client, logger) -const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec), chapter_reader, new SchemaValidator(spec, logger)) +const chapter_evaluator = new ChapterEvaluator(new OperationLocator(spec), chapter_reader, new SchemaValidator(spec, logger), logger) const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader) const story_validator = new StoryValidator() const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator) diff --git a/tools/tests/tester/fixtures/evals/passed.yaml b/tools/tests/tester/fixtures/evals/passed.yaml index a1e264847..ddf64b196 100644 --- a/tools/tests/tester/fixtures/evals/passed.yaml +++ b/tools/tests/tester/fixtures/evals/passed.yaml @@ -69,6 +69,22 @@ chapters: result: PASSED payload_schema: result: PASSED + - title: This GET /_cat/health chapter returns application/cbor and should pass. + overall: + result: PASSED + request: + parameters: + format: + result: PASSED + request_body: + result: PASSED + response: + status: + result: PASSED + payload_body: + result: PASSED + payload_schema: + result: PASSED epilogues: - title: DELETE /books overall: diff --git a/tools/tests/tester/fixtures/specs/excerpt.yaml b/tools/tests/tester/fixtures/specs/excerpt.yaml index 136f08511..7254f0ccd 100644 --- a/tools/tests/tester/fixtures/specs/excerpt.yaml +++ b/tools/tests/tester/fixtures/specs/excerpt.yaml @@ -101,6 +101,11 @@ components: type: array items: type: object + application/cbor: + schema: + type: array + items: + type: object indices.delete@200: description: '' content: diff --git a/tools/tests/tester/fixtures/stories/passed.yaml b/tools/tests/tester/fixtures/stories/passed.yaml index c4a3c5c2b..08fe26349 100644 --- a/tools/tests/tester/fixtures/stories/passed.yaml +++ b/tools/tests/tester/fixtures/stories/passed.yaml @@ -34,3 +34,13 @@ chapters: content_type: application/yaml payload: - node.total: '1' + - synopsis: This GET /_cat/health chapter returns application/cbor and should pass. + path: /_cat/health + parameters: + format: cbor + method: GET + response: + status: 200 + content_type: application/cbor + payload: + - node.total: '1' diff --git a/tools/tests/tester/helpers.ts b/tools/tests/tester/helpers.ts index 4f59cf741..0637fd0b2 100644 --- a/tools/tests/tester/helpers.ts +++ b/tools/tests/tester/helpers.ts @@ -46,7 +46,7 @@ export function construct_tester_components (spec_path: string): { }) const chapter_reader = new ChapterReader(opensearch_http_client, logger) const schema_validator = new SchemaValidator(specification, logger) - const chapter_evaluator = new ChapterEvaluator(operation_locator, chapter_reader, schema_validator) + const chapter_evaluator = new ChapterEvaluator(operation_locator, chapter_reader, schema_validator, logger) const supplemental_chapter_evaluator = new SupplementalChapterEvaluator(chapter_reader) const story_validator = new StoryValidator() const story_evaluator = new StoryEvaluator(chapter_evaluator, supplemental_chapter_evaluator)