From 87e0693f3bd319ce3c15884350d584be9700804e Mon Sep 17 00:00:00 2001 From: Theo Nam Truong Date: Sat, 4 May 2024 06:46:10 -0600 Subject: [PATCH] Added validation for _info.yaml (#281) DRY'ed JSON schema validation logic Signed-off-by: Theo Truong --- json_schemas/_info.schema.yaml | 44 +++++++++++++++++++ ...aml => _superseded_operations.schema.yaml} | 1 + spec/_info.yaml | 5 ++- spec/_superseded_operations.yaml | 2 +- tools/helpers.ts | 6 ++- tools/linter/SpecValidator.ts | 10 +++-- tools/linter/components/InfoFile.ts | 5 +++ .../components/SupersededOperationsFile.ts | 19 +------- tools/linter/components/base/FileValidator.ts | 20 ++++++++- tools/merger/OpenApiMerger.ts | 2 +- tools/package-lock.json | 17 +++++++ tools/package.json | 1 + tools/test/linter/InfoFile.test.ts | 12 +++++ .../linter/SupersededOperationsFile.test.ts | 2 +- tools/test/linter/fixtures/_info.yaml | 2 + .../fixtures/_superseded_operations.yaml | 2 +- tools/test/linter/fixtures/empty/_info.yaml | 4 ++ .../empty/_superseded_operations.yaml | 2 +- tools/test/merger/fixtures/spec/_info.yaml | 2 + 19 files changed, 127 insertions(+), 31 deletions(-) create mode 100644 json_schemas/_info.schema.yaml rename json_schemas/{_superseded_operations.yaml => _superseded_operations.schema.yaml} (99%) create mode 100644 tools/linter/components/InfoFile.ts create mode 100644 tools/test/linter/InfoFile.test.ts create mode 100644 tools/test/linter/fixtures/_info.yaml create mode 100644 tools/test/linter/fixtures/empty/_info.yaml diff --git a/json_schemas/_info.schema.yaml b/json_schemas/_info.schema.yaml new file mode 100644 index 000000000..77a2257b7 --- /dev/null +++ b/json_schemas/_info.schema.yaml @@ -0,0 +1,44 @@ +$schema: http://json-schema.org/draft-07/schema# + +type: object +properties: + title: + type: string + summary: + type: string + description: + type: string + termsOfService: + type: string + format: uri + contact: + $comment: https://spec.openapis.org/oas/v3.1.0#contact-object + type: object + properties: + name: + type: string + url: + type: string + format: uri + email: + type: string + format: email + license: + $comment: https://spec.openapis.org/oas/v3.1.0#license-object + type: object + properties: + name: + type: string + identifier: + type: string + url: + type: string + format: uri + required: + - name + version: + type: string +required: + - title + - version + - $schema \ No newline at end of file diff --git a/json_schemas/_superseded_operations.yaml b/json_schemas/_superseded_operations.schema.yaml similarity index 99% rename from json_schemas/_superseded_operations.yaml rename to json_schemas/_superseded_operations.schema.yaml index ed298d7f3..a49f6671e 100644 --- a/json_schemas/_superseded_operations.yaml +++ b/json_schemas/_superseded_operations.schema.yaml @@ -1,4 +1,5 @@ $schema: http://json-schema.org/draft-07/schema# + type: object patternProperties: ^\$schema$: diff --git a/spec/_info.yaml b/spec/_info.yaml index 9da26bfa0..f71d35dd0 100644 --- a/spec/_info.yaml +++ b/spec/_info.yaml @@ -1,3 +1,4 @@ -title: OpenSearch API -description: OpenSearch API +$schema: ../json_schemas/_info.schema.yaml + +title: OpenSearch API Specification version: 1.0.0 \ No newline at end of file diff --git a/spec/_superseded_operations.yaml b/spec/_superseded_operations.yaml index 3996da336..c5180f076 100644 --- a/spec/_superseded_operations.yaml +++ b/spec/_superseded_operations.yaml @@ -1,4 +1,4 @@ -$schema: ../json_schemas/_superseded_operations.yaml +$schema: ../json_schemas/_superseded_operations.schema.yaml /_opendistro/_alerting/destinations: superseded_by: /_plugins/_alerting/destinations diff --git a/tools/helpers.ts b/tools/helpers.ts index 09e6fcb6f..cdbb5af53 100644 --- a/tools/helpers.ts +++ b/tools/helpers.ts @@ -43,8 +43,10 @@ export function sort_by_keys (obj: Record, priorities: string[] = [ }) } -export function read_yaml (file_path: string): Record { - return YAML.parse(fs.readFileSync(file_path, 'utf8')) +export function read_yaml (file_path: string, exclude_schema: boolean = false): Record { + const doc = YAML.parse(fs.readFileSync(file_path, 'utf8')) + if (exclude_schema) delete doc.$schema + return doc } export function write_yaml (file_path: string, content: Record): void { diff --git a/tools/linter/SpecValidator.ts b/tools/linter/SpecValidator.ts index a9ea98f14..d479da16e 100644 --- a/tools/linter/SpecValidator.ts +++ b/tools/linter/SpecValidator.ts @@ -3,15 +3,18 @@ import NamespacesFolder from './components/NamespacesFolder' import { type ValidationError } from '../types' import SchemaRefsValidator from './SchemaRefsValidator' import SupersededOperationsFile from './components/SupersededOperationsFile' +import InfoFile from './components/InfoFile' export default class SpecValidator { - superseded_ops_files: SupersededOperationsFile + superseded_ops_file: SupersededOperationsFile + info_file: InfoFile namespaces_folder: NamespacesFolder schemas_folder: SchemasFolder schema_refs_validator: SchemaRefsValidator constructor (root_folder: string) { - this.superseded_ops_files = new SupersededOperationsFile(`${root_folder}/_superseded_operations.yaml`) + this.superseded_ops_file = new SupersededOperationsFile(`${root_folder}/_superseded_operations.yaml`) + this.info_file = new InfoFile(`${root_folder}/_info.yaml`) this.namespaces_folder = new NamespacesFolder(`${root_folder}/namespaces`) this.schemas_folder = new SchemasFolder(`${root_folder}/schemas`) this.schema_refs_validator = new SchemaRefsValidator(this.namespaces_folder, this.schemas_folder) @@ -26,7 +29,8 @@ export default class SpecValidator { return [ ...this.schema_refs_validator.validate(), - ...this.superseded_ops_files.validate() + ...this.superseded_ops_file.validate(), + ...this.info_file.validate() ] } } diff --git a/tools/linter/components/InfoFile.ts b/tools/linter/components/InfoFile.ts new file mode 100644 index 000000000..791beda41 --- /dev/null +++ b/tools/linter/components/InfoFile.ts @@ -0,0 +1,5 @@ +import FileValidator from './base/FileValidator' + +export default class InfoFile extends FileValidator { + has_json_schema = true +} diff --git a/tools/linter/components/SupersededOperationsFile.ts b/tools/linter/components/SupersededOperationsFile.ts index 335a897ff..3591ede8e 100644 --- a/tools/linter/components/SupersededOperationsFile.ts +++ b/tools/linter/components/SupersededOperationsFile.ts @@ -1,22 +1,5 @@ import FileValidator from './base/FileValidator' -import ajv from 'ajv' -import { type ValidationError } from '../../types' -import { read_yaml } from '../../helpers' export default class SupersededOperationsFile extends FileValidator { - readonly JSON_SCHEMA_PATH = '../json_schemas/_superseded_operations.yaml' - - validate (): ValidationError[] { - return [ - this.validate_json_schema() - ].filter(e => e) as ValidationError[] - } - - validate_json_schema (): ValidationError | undefined { - const schema = read_yaml(this.JSON_SCHEMA_PATH) - const validator = (new ajv()).compile(schema) - if (!validator(this.spec())) { - return this.error(`File content does not match JSON schema found in '${this.JSON_SCHEMA_PATH}':\n ${JSON.stringify(validator.errors, null, 2)}`) - } - } + has_json_schema = true } diff --git a/tools/linter/components/base/FileValidator.ts b/tools/linter/components/base/FileValidator.ts index f2d777216..a0e0fa7c7 100644 --- a/tools/linter/components/base/FileValidator.ts +++ b/tools/linter/components/base/FileValidator.ts @@ -2,9 +2,12 @@ import ValidatorBase from './ValidatorBase' import { type ValidationError } from '../../../types' import { type OpenAPIV3 } from 'openapi-types' import { read_yaml } from '../../../helpers' +import AJV from 'ajv' +import addFormats from 'ajv-formats' export default class FileValidator extends ValidatorBase { file_path: string + has_json_schema: boolean = false protected _spec: OpenAPIV3.Document | undefined constructor (file_path: string) { @@ -23,11 +26,13 @@ export default class FileValidator extends ValidatorBase { if (extension_error) return [extension_error] const yaml_error = this.validate_yaml() if (yaml_error) return [yaml_error] + const json_schema_error = this.validate_json_schema() + if (json_schema_error) return [json_schema_error] return this.validate_file() } validate_file (): ValidationError[] { - throw new Error('Method not implemented.') + return [] } validate_extension (): ValidationError | undefined { @@ -41,4 +46,17 @@ export default class FileValidator extends ValidatorBase { return this.error('Unable to read or parse YAML.', 'File Content') } } + + validate_json_schema (): ValidationError | undefined { + if (!this.has_json_schema) return + const json_schema_path: string = (this.spec() as any).$schema ?? '' + if (json_schema_path === '') return this.error('JSON Schema is required but not found in this file.', '$schema') + const schema = read_yaml(json_schema_path) + const ajv = new AJV({ schemaId: 'id' }) + addFormats(ajv) + const validator = ajv.compile(schema) + if (!validator(this.spec())) { + return this.error(`File content does not match JSON schema found in '${json_schema_path}':\n ${JSON.stringify(validator.errors, null, 2)}`) + } + } } diff --git a/tools/merger/OpenApiMerger.ts b/tools/merger/OpenApiMerger.ts index 7054eac12..0bb0be431 100644 --- a/tools/merger/OpenApiMerger.ts +++ b/tools/merger/OpenApiMerger.ts @@ -17,7 +17,7 @@ export default class OpenApiMerger { this.root_folder = fs.realpathSync(root_folder) this.spec = { openapi: '3.1.0', - info: read_yaml(`${this.root_folder}/_info.yaml`), + info: read_yaml(`${this.root_folder}/_info.yaml`, true), paths: {}, components: { parameters: {}, diff --git a/tools/package-lock.json b/tools/package-lock.json index baf75ba26..20ab5a4d4 100644 --- a/tools/package-lock.json +++ b/tools/package-lock.json @@ -13,6 +13,7 @@ "@types/lodash": "^4.14.202", "@types/node": "^20.10.3", "ajv": "^8.13.0", + "ajv-formats": "^3.0.1", "lodash": "^4.17.21", "ts-node": "^10.9.1", "typescript": "^5.4.5", @@ -1831,6 +1832,22 @@ } } }, + "node_modules/ajv-formats": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-3.0.1.tgz", + "integrity": "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, "node_modules/ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", diff --git a/tools/package.json b/tools/package.json index 0e7bbbfde..23bafa58d 100644 --- a/tools/package.json +++ b/tools/package.json @@ -15,6 +15,7 @@ "@types/lodash": "^4.14.202", "@types/node": "^20.10.3", "ajv": "^8.13.0", + "ajv-formats": "^3.0.1", "lodash": "^4.17.21", "ts-node": "^10.9.1", "typescript": "^5.4.5", diff --git a/tools/test/linter/InfoFile.test.ts b/tools/test/linter/InfoFile.test.ts new file mode 100644 index 000000000..a0a7652bd --- /dev/null +++ b/tools/test/linter/InfoFile.test.ts @@ -0,0 +1,12 @@ +import InfoFile from '../../linter/components/InfoFile' + +test('validate()', () => { + const validator = new InfoFile('./test/linter/fixtures/_info.yaml') + expect(validator.validate()).toEqual([ + { + file: 'fixtures/_info.yaml', + location: '$schema', + message: 'JSON Schema is required but not found in this file.' + } + ]) +}) diff --git a/tools/test/linter/SupersededOperationsFile.test.ts b/tools/test/linter/SupersededOperationsFile.test.ts index be93ea7a0..f5c6a7e3a 100644 --- a/tools/test/linter/SupersededOperationsFile.test.ts +++ b/tools/test/linter/SupersededOperationsFile.test.ts @@ -5,7 +5,7 @@ test('validate()', () => { expect(validator.validate()).toEqual([ { file: 'fixtures/_superseded_operations.yaml', - message: "File content does not match JSON schema found in '../json_schemas/_superseded_operations.yaml':\n [\n {\n \"instancePath\": \"/~1hello~1world/operations/1\",\n \"schemaPath\": \"#/patternProperties/%5E~1/properties/operations/items/enum\",\n \"keyword\": \"enum\",\n \"params\": {\n \"allowedValues\": [\n \"GET\",\n \"POST\",\n \"PUT\",\n \"DELETE\",\n \"HEAD\",\n \"OPTIONS\",\n \"PATCH\"\n ]\n },\n \"message\": \"must be equal to one of the allowed values\"\n }\n]" + message: "File content does not match JSON schema found in '../json_schemas/_superseded_operations.schema.yaml':\n [\n {\n \"instancePath\": \"/~1hello~1world/operations/1\",\n \"schemaPath\": \"#/patternProperties/%5E~1/properties/operations/items/enum\",\n \"keyword\": \"enum\",\n \"params\": {\n \"allowedValues\": [\n \"GET\",\n \"POST\",\n \"PUT\",\n \"DELETE\",\n \"HEAD\",\n \"OPTIONS\",\n \"PATCH\"\n ]\n },\n \"message\": \"must be equal to one of the allowed values\"\n }\n]" } ]) }) diff --git a/tools/test/linter/fixtures/_info.yaml b/tools/test/linter/fixtures/_info.yaml new file mode 100644 index 000000000..29b411f57 --- /dev/null +++ b/tools/test/linter/fixtures/_info.yaml @@ -0,0 +1,2 @@ +title: OpenSearch API Specification +version: 1.0.0 \ No newline at end of file diff --git a/tools/test/linter/fixtures/_superseded_operations.yaml b/tools/test/linter/fixtures/_superseded_operations.yaml index d264c0681..20c92eec9 100644 --- a/tools/test/linter/fixtures/_superseded_operations.yaml +++ b/tools/test/linter/fixtures/_superseded_operations.yaml @@ -1,4 +1,4 @@ -$schema: ../../../../json_schemas/_superseded_operations.yaml +$schema: ../json_schemas/_superseded_operations.schema.yaml /hello/world: superseded_by: /goodbye/world diff --git a/tools/test/linter/fixtures/empty/_info.yaml b/tools/test/linter/fixtures/empty/_info.yaml new file mode 100644 index 000000000..66d1923d4 --- /dev/null +++ b/tools/test/linter/fixtures/empty/_info.yaml @@ -0,0 +1,4 @@ +$schema: ../json_schemas/_info.schema.yaml + +title: '' +version: '' \ No newline at end of file diff --git a/tools/test/linter/fixtures/empty/_superseded_operations.yaml b/tools/test/linter/fixtures/empty/_superseded_operations.yaml index ac2175ea5..6f143f79b 100644 --- a/tools/test/linter/fixtures/empty/_superseded_operations.yaml +++ b/tools/test/linter/fixtures/empty/_superseded_operations.yaml @@ -1 +1 @@ -$schema: ../../../../../json_schemas/_superseded_operations.yaml \ No newline at end of file +$schema: ../json_schemas/_superseded_operations.schema.yaml \ No newline at end of file diff --git a/tools/test/merger/fixtures/spec/_info.yaml b/tools/test/merger/fixtures/spec/_info.yaml index 9da26bfa0..721aed8c9 100644 --- a/tools/test/merger/fixtures/spec/_info.yaml +++ b/tools/test/merger/fixtures/spec/_info.yaml @@ -1,3 +1,5 @@ +$schema: should-be-ignored + title: OpenSearch API description: OpenSearch API version: 1.0.0 \ No newline at end of file