From 90132054984b6a93089aeafd9ce6ad93c386eab7 Mon Sep 17 00:00:00 2001 From: DJ Madeira Date: Tue, 21 May 2024 19:18:21 -0400 Subject: [PATCH] Add Transforms API Schema (#288) * Add Transforms API Schema Signed-off-by: djmadeira --- spec/namespaces/transforms.yaml | 264 ++++++++++++++++++ spec/schemas/transforms._common.yaml | 205 ++++++++++++++ tools/src/linter/components/NamespaceFile.ts | 4 +- tools/src/linter/components/Operation.ts | 2 +- tools/tests/linter/NamespaceFile.test.ts | 4 +- .../namespaces/invalid_components.yaml | 8 +- 6 files changed, 478 insertions(+), 9 deletions(-) create mode 100644 spec/namespaces/transforms.yaml create mode 100644 spec/schemas/transforms._common.yaml diff --git a/spec/namespaces/transforms.yaml b/spec/namespaces/transforms.yaml new file mode 100644 index 000000000..c7335c5f9 --- /dev/null +++ b/spec/namespaces/transforms.yaml @@ -0,0 +1,264 @@ +openapi: 3.1.0 +info: + title: OpenSearch Index Transforms API + description: OpenSearch Index Transforms API + version: 1.0.0 +paths: + /_plugins/_transform: + get: + operationId: transforms.search.0 + x-operation-group: transforms.search + x-version-added: 1.0 + description: Returns the details of all transform jobs. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/index-transforms/transforms-apis/#get-a-transform-jobs-details + parameters: + - $ref: '#/components/parameters/transforms.search::query.size' + - $ref: '#/components/parameters/transforms.search::query.from' + - $ref: '#/components/parameters/transforms.search::query.search' + - $ref: '#/components/parameters/transforms.search::query.sortField' + - $ref: '#/components/parameters/transforms.search::query.sortDirection' + responses: + '200': + $ref: '#/components/responses/transforms.search@200' + /_plugins/_transform/{id}: + get: + operationId: transforms.get.0 + x-operation-group: transforms.get + x-version-added: 1.0 + description: Returns the status and metadata of a transform job. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/index-transforms/transforms-apis/#get-a-transform-jobs-details + parameters: + - $ref: '#/components/parameters/transforms.get::path.id' + responses: + '200': + $ref: '#/components/responses/transforms.get@200' + put: + operationId: transforms.put.0 + x-operation-group: transforms.put + x-version-added: 1.0 + description: Create an index transform, or update a transform if if_seq_no and if_primary_term are provided. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/index-transforms/transforms-apis/#create-a-transform-job + parameters: + - $ref: '#/components/parameters/transforms.put::path.id' + - $ref: '#/components/parameters/transforms.put::query.if_seq_no' + - $ref: '#/components/parameters/transforms.put::query.if_primary_term' + requestBody: + $ref: '#/components/requestBodies/transforms.put' + responses: + '200': + $ref: '#/components/responses/transforms.put@200' + delete: + operationId: transforms.delete.0 + x-operation-group: transforms.delete + x-version-added: 1.0 + description: Delete an index transform. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/index-transforms/transforms-apis/#delete-a-transform-job + parameters: + - $ref: '#/components/parameters/transforms.delete::path.id' + responses: + '200': + $ref: '#/components/responses/transforms.delete@200' + /_plugins/_transform/{id}/_start: + post: + operationId: transforms.start.0 + x-operation-group: transforms.start + x-version-added: 1.0 + description: Start transform. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/index-transforms/transforms-apis/#start-a-transform-job + parameters: + - $ref: '#/components/parameters/transforms.start::path.id' + responses: + '200': + $ref: '#/components/responses/transforms.start@200' + /_plugins/_transform/{id}/_stop: + post: + operationId: transforms.stop.0 + x-operation-group: transforms.stop + x-version-added: 1.0 + description: stop transform. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/index-transforms/transforms-apis/#stop-a-transform-job + parameters: + - $ref: '#/components/parameters/transforms.stop::path.id' + responses: + '200': + $ref: '#/components/responses/transforms.stop@200' + /_plugins/_transform/{id}/_explain: + get: + operationId: transforms.explain.0 + x-operation-group: transforms.explain + x-version-added: 1.0 + description: Returns the status and metadata of a transform job. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/index-transforms/transforms-apis/#get-the-status-of-a-transform-job + parameters: + - $ref: '#/components/parameters/transforms.explain::path.id' + responses: + '200': + $ref: '#/components/responses/transforms.explain@200' + /_plugins/_transform/_preview: + get: + operationId: transforms.preview.0 + x-operation-group: transforms.preview + x-version-added: 1.0 + description: Returns a preview of what a transformed index would look like. + externalDocs: + url: https://opensearch.org/docs/latest/im-plugin/index-transforms/transforms-apis/#preview-a-transform-jobs-results + responses: + '200': + $ref: '#/components/responses/transforms.preview@200' +components: + requestBodies: + transforms.put: + content: + application/json: + schema: + $ref: '../schemas/transforms._common.yaml#/components/schemas/Transform' + responses: + transforms.search@200: + content: + application/json: + schema: + $ref: '../schemas/transforms._common.yaml#/components/schemas/TransformsResponse' + transforms.get@200: + content: + application/json: + schema: + $ref: '../schemas/transforms._common.yaml#/components/schemas/TransformEntity' + transforms.put@200: + content: + application/json: + schema: + $ref: '../schemas/transforms._common.yaml#/components/schemas/TransformEntity' + transforms.delete@200: + description: '' + transforms.start@200: + content: + application/json: + schema: + $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' + transforms.stop@200: + content: + application/json: + schema: + $ref: '../schemas/_common.yaml#/components/schemas/AcknowledgedResponseBase' + transforms.explain@200: + content: + application/json: + schema: + $ref: '../schemas/transforms._common.yaml#/components/schemas/ExplainResponse' + transforms.preview@200: + content: + application/json: + schema: + $ref: '../schemas/transforms._common.yaml#/components/schemas/Preview' + parameters: + transforms.get::path.id: + name: id + in: path + description: Transform to access + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Id' + style: simple + transforms.put::path.id: + name: id + in: path + description: Transform to create/update + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Id' + style: simple + transforms.delete::path.id: + name: id + in: path + description: Transform to delete + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Id' + style: simple + transforms.start::path.id: + name: id + in: path + description: Transform to start + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Id' + style: simple + transforms.stop::path.id: + name: id + in: path + description: Transform to stop + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Id' + style: simple + transforms.explain::path.id: + name: id + in: path + description: Transform to explain + required: true + schema: + $ref: '../schemas/_common.yaml#/components/schemas/Id' + style: simple + transforms.put::query.if_seq_no: + name: if_seq_no + in: query + description: Only perform the operation if the document has this sequence number. + required: false + schema: + $ref: '../schemas/_common.yaml#/components/schemas/SequenceNumber' + style: form + transforms.put::query.if_primary_term: + name: if_primary_term + in: query + description: Only perform the operation if the document has this primary term. + required: false + schema: + type: number + style: form + transforms.search::query.size: + name: size + in: query + description: Specifies the number of transforms to return. Default is 10. + required: false + schema: + type: number + style: form + transforms.search::query.from: + name: from + in: query + description: The starting transform to return. Default is 0. + required: false + schema: + type: number + style: form + transforms.search::query.search: + name: search + in: query + description: The search term to use to filter results. + required: false + schema: + type: string + style: form + transforms.search::query.sortField: + name: sortField + in: query + description: The field to sort results with. + required: false + schema: + type: string + style: form + transforms.search::query.sortDirection: + name: sortDirection + in: query + description: Specifies the direction to sort results in. Can be ASC or DESC. Default is ASC. + required: false + schema: + type: string + style: form diff --git a/spec/schemas/transforms._common.yaml b/spec/schemas/transforms._common.yaml new file mode 100644 index 000000000..5d44ef5d9 --- /dev/null +++ b/spec/schemas/transforms._common.yaml @@ -0,0 +1,205 @@ +openapi: 3.1.0 +info: + title: Schemas of transforms._common category + description: Schemas of transforms._common category + version: 1.0.0 +components: + schemas: + TransformEntity: + type: object + properties: + _id: + type: string + _seqNo: + type: number + _primaryTerm: + type: number + transform: + $ref: '#/components/schemas/Transform' + TransformsResponse: + type: object + properties: + total_transforms: + type: number + transforms: + type: array + items: + $ref: '#/components/schemas/TransformEntity' + Transform: + type: object + properties: + transform_id: + type: string + schema_version: + type: number + continuous: + type: boolean + schedule: + $ref: '#/components/schemas/Schedule' + metadata_id: + type: string + updated_at: + type: string + enabled: + type: boolean + enabled_at: + type: number + description: + type: string + source_index: + type: string + data_selection_query: + $ref: '_common.query_dsl.yaml#/components/schemas/QueryContainer' + target_index: + type: string + roles: + type: array + items: + type: string + page_size: + type: number + groups: + type: array + items: + $ref: '#/components/schemas/GroupsConfigItem' + aggregations: + type: array + items: + $ref: '#/components/schemas/MetricsConfigItem' + ExplainResponse: + type: object + additionalProperties: + $ref: '#/components/schemas/Explain' + Explain: + type: object + properties: + metadata_id: + type: string + transform_metadata: + $ref: '#/components/schemas/TransformMetadata' + Preview: + type: object + properties: + documents: + type: array + items: + type: object + Schedule: + type: object + properties: + interval: + $ref: '#/components/schemas/ScheduleInterval' + required: + - interval + ScheduleInterval: + type: object + properties: + start_time: + type: number + period: + type: number + unit: + type: string + GroupsConfigItem: + type: object + properties: + histogram: + $ref: '#/components/schemas/HistogramGroup' + date_histogram: + $ref: '#/components/schemas/DateHistogramGroup' + terms: + $ref: '#/components/schemas/TermsGroup' + minProperties: 1 + maxProperties: 1 + DateHistogramGroup: + type: object + properties: + fixed_interval: + type: string + calendar_interval: + type: string + timezone: + type: string + source_field: + type: string + target_field: + type: string + HistogramGroup: + type: object + properties: + source_field: + type: string + target_field: + type: string + interval: + type: string + TermsGroup: + type: object + properties: + source_field: + type: string + target_field: + type: string + MetricsConfigItem: + type: object + properties: + source_field: + type: string + target_field: + type: string + metrics: + type: array + items: + $ref: '#/components/schemas/MetricsConfigMetrics' + MetricsConfigMetrics: + type: object + properties: + avg: + type: object + sum: + type: object + max: + type: object + min: + type: object + value_count: + type: object + minProperties: 1 + maxProperties: 1 + ContinuousStats: + type: object + properties: + last_timestamp: + type: number + documents_behind: + type: object + additionalProperties: + type: number + ExplainStats: + type: object + properties: + pages_processed: + type: number + documents_processed: + type: number + documents_indexed: + type: number + index_time_in_millis: + type: number + search_time_in_millis: + type: number + TransformMetadata: + type: object + properties: + continuous_stats: + $ref: '#/components/schemas/ContinuousStats' + transform_id: + type: string + last_updated_at: + type: number + status: + type: string + failure_reason: + type: string + stats: + $ref: '#/components/schemas/ExplainStats' diff --git a/tools/src/linter/components/NamespaceFile.ts b/tools/src/linter/components/NamespaceFile.ts index 1373b72f9..8bfe67f17 100644 --- a/tools/src/linter/components/NamespaceFile.ts +++ b/tools/src/linter/components/NamespaceFile.ts @@ -93,9 +93,9 @@ export default class NamespaceFile extends FileValidator { `Parameter component '${name}' must be named '${expected}' since it is a ${p.in} parameter named '${p.name}'.`, `#/components/parameters/#${name}`) } - if (!p.name.match(/^[a-z0-9._]+$/)) { + if (!p.name.match(/^[a-zA-Z0-9._]+$/)) { return this.error( - `Invalid parameter name '${p.name}'. A parameter's name can only contain lower-cased alphanumerics, underscores, and periods.`, + `Invalid parameter name '${p.name}'. A parameter's name can only contain alphanumerics, underscores, and periods.`, `#/components/parameters/#${name}`) } }).filter((e) => e) as ValidationError[] diff --git a/tools/src/linter/components/Operation.ts b/tools/src/linter/components/Operation.ts index 6a9f2ead6..99a93c3c1 100644 --- a/tools/src/linter/components/Operation.ts +++ b/tools/src/linter/components/Operation.ts @@ -84,7 +84,7 @@ export default class Operation extends ValidatorBase { validate_parameters (): ValidationError | undefined { const parameters = this.spec.parameters if (!parameters) return - const regex = new RegExp(`^#/components/parameters/${this.group_regex}::((path)|(query))\\.[a-z0-9_.]+$`) + const regex = new RegExp(`^#/components/parameters/${this.group_regex}::((path)|(query))\\.[a-zA-Z0-9_.]+$`) for (const parameter of parameters) { if (!regex.test(parameter.$ref)) { return this.error('Every parameter must be a reference object to \'#/components/parameters/{x-operation-group}::{path|query}.{parameter_name}\'.') } } diff --git a/tools/tests/linter/NamespaceFile.test.ts b/tools/tests/linter/NamespaceFile.test.ts index 3303bbc87..51fe30339 100644 --- a/tools/tests/linter/NamespaceFile.test.ts +++ b/tools/tests/linter/NamespaceFile.test.ts @@ -68,8 +68,8 @@ test('validate_parameter_refs()', () => { expect(validator.validate_parameter_refs()).toEqual([ { file: 'namespaces/invalid_components.yaml', - location: '#/components/parameters/#indices.create::query.ExpandWildcards', - message: "Invalid parameter name 'ExpandWildcards'. A parameter's name can only contain lower-cased alphanumerics, underscores, and periods." + location: '#/components/parameters/#indices.create::query.ExpandWildcard$', + message: "Invalid parameter name 'ExpandWildcard$'. A parameter's name can only contain alphanumerics, underscores, and periods." }, { file: 'namespaces/invalid_components.yaml', diff --git a/tools/tests/linter/fixtures/file_validators/namespaces/invalid_components.yaml b/tools/tests/linter/fixtures/file_validators/namespaces/invalid_components.yaml index 4923d42aa..4d248e5e2 100644 --- a/tools/tests/linter/fixtures/file_validators/namespaces/invalid_components.yaml +++ b/tools/tests/linter/fixtures/file_validators/namespaces/invalid_components.yaml @@ -6,7 +6,7 @@ paths: $ref: '#/components/responses/indices.create@200' parameters: - $ref: '#/components/parameters/indices.create::path.index' - - $ref: '#/components/parameters/indices.create::query.ExpandWildcards' + - $ref: '#/components/parameters/indices.create::query.ExpandWildcard$' - $ref: '#/components/parameters/indices.create::query.pretty' components: @@ -16,9 +16,9 @@ components: indices.create::path.index: name: index in: path - indices.create::query.ExpandWildcards: - name: ExpandWildcards + indices.create::query.ExpandWildcard$: + name: ExpandWildcard$ in: query indices.create::query.h: name: v - in: query \ No newline at end of file + in: query