From 2994879ce4e2697e710dc474fba822e0e09131e5 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Thu, 12 Dec 2024 12:21:50 -0500 Subject: [PATCH 01/14] Added test for GET /_settings/{name}. (#727) Signed-off-by: dblock Signed-off-by: Nathalie Jonathan --- tests/default/_core/settings.yaml | 58 +++++++++++++++++++++++++++++ tests/default/indices/settings.yaml | 34 ----------------- 2 files changed, 58 insertions(+), 34 deletions(-) create mode 100644 tests/default/_core/settings.yaml diff --git a/tests/default/_core/settings.yaml b/tests/default/_core/settings.yaml new file mode 100644 index 000000000..919b88fcc --- /dev/null +++ b/tests/default/_core/settings.yaml @@ -0,0 +1,58 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test settings. +prologues: + - path: /movies + method: PUT + - path: /movies/_settings + method: PUT + request: + payload: + settings: + index.blocks.write: true + index: + number_of_replicas: 4 +epilogues: + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Get settings. + distributions: + excluded: + - amazon-managed + - amazon-serverless + path: /_settings + method: GET + parameters: + allow_no_indices: true + expand_wildcards: all + flat_settings: true + include_defaults: true + ignore_unavailable: true + local: true + response: + status: 200 + - synopsis: Get settings (cluster_manager_timeout). + distributions: + excluded: + - amazon-managed + - amazon-serverless + path: /_settings + method: GET + version: '>= 2.0' + parameters: + allow_no_indices: true + expand_wildcards: all + flat_settings: true + include_defaults: true + ignore_unavailable: true + local: true + cluster_manager_timeout: 1s + response: + status: 200 + - synopsis: Get settings by name. + path: /_settings/{name} + method: GET + parameters: + name: index.number_of_replicas \ No newline at end of file diff --git a/tests/default/indices/settings.yaml b/tests/default/indices/settings.yaml index 8513ffaa6..0ac83462b 100644 --- a/tests/default/indices/settings.yaml +++ b/tests/default/indices/settings.yaml @@ -10,40 +10,6 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Get global settings. - distributions: - excluded: - - amazon-managed - - amazon-serverless - path: /_settings - method: GET - parameters: - allow_no_indices: true - expand_wildcards: all - flat_settings: true - include_defaults: true - ignore_unavailable: true - local: true - response: - status: 200 - - synopsis: Get global settings (cluster_manager_timeout). - distributions: - excluded: - - amazon-managed - - amazon-serverless - path: /_settings - method: GET - version: '>= 2.0' - parameters: - allow_no_indices: true - expand_wildcards: all - flat_settings: true - include_defaults: true - ignore_unavailable: true - local: true - cluster_manager_timeout: 1s - response: - status: 200 - synopsis: Write a setting to an index. path: /{index}/_settings method: PUT From 14cc701835ef2366fd2254f707cd05f428a3e9d2 Mon Sep 17 00:00:00 2001 From: Nathalie Jonathan Date: Fri, 13 Dec 2024 13:48:21 -0800 Subject: [PATCH 02/14] Added ML Model APIs (get, search, update model) to the spec along with the tests Signed-off-by: Nathalie Jonathan --- CHANGELOG.md | 1 + spec/namespaces/ml.yaml | 81 ++++++++++++++++++ spec/schemas/ml._common.yaml | 146 ++++++++++++++++++++++++++++++++ tests/plugins/ml/ml/models.yaml | 31 +++++++ 4 files changed, 259 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e6fdd6040..be50b8000 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added response schema for `PUT` and `DELETE /_plugins/_transform/{id}` ([#722](https://github.com/opensearch-project/opensearch-api-specification/pull/716)) - Added response schema for `GET /_plugins/_knn/warmup/{index}` ([#717](https://github.com/opensearch-project/opensearch-api-specification/pull/717)) - Added support for multiple test verbs ([#724](https://github.com/opensearch-project/opensearch-api-specification/pull/724)) +- Added ML Model APIs `GET /_plugins/_ml/models/{}`, `POST /_plugins/_ml/models/_search`, and `PUT /_plugins/_ml/models/{}` to the spec along with the tests. ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index 67ec2460a..1148c496e 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -44,6 +44,26 @@ paths: '200': $ref: '#/components/responses/ml.register_model@200' /_plugins/_ml/models/{model_id}: + get: + operationId: ml.get_model.0 + x-operation-group: ml.get_model + description: Retrieves a model. + parameters: + - $ref: '#/components/parameters/ml.get_model::path.model_id' + responses: + '200': + $ref: '#/components/responses/ml.get_model@200' + put: + operationId: ml.update_model.0 + x-operation-group: ml.update_model + description: Updates a model. + parameters: + - $ref: '#/components/parameters/ml.update_model::path.model_id' + requestBody: + $ref: '#/components/requestBodies/ml.update_model' + responses: + '200': + $ref: '#/components/responses/ml.update_model@200' delete: operationId: ml.delete_model.0 x-operation-group: ml.delete_model @@ -102,6 +122,15 @@ paths: responses: '200': $ref: '#/components/responses/ml.search_models@200' + post: + operationId: ml.search_models.0 + x-operation-group: ml.search_models + description: Searches for models. + requestBody: + $ref: '#/components/requestBodies/ml.search_models' + responses: + '200': + $ref: '#/components/responses/ml.search_models@200' /_plugins/_ml/connectors/_create: post: operationId: ml.create_connector.0 @@ -212,6 +241,36 @@ components: required: - query - size + ml.update_model: + content: + application/json: + schema: + type: object + properties: + connector: + type: object + description: The connector to use for the model. + connector_id: + type: string + description: The connector ID. + is_enabled: + type: boolean + description: Whether the model is enabled. + description: + type: string + description: The model description. + model_config: + $ref: '../schemas/ml._common.yaml#/components/schemas/ModelConfig' + name: + type: string + description: The model name. + rate_limiter: + $ref: '../schemas/ml._common.yaml#/components/schemas/RateLimiter' + guardrails: + $ref: '../schemas/ml._common.yaml#/components/schemas/Guardrails' + interface: + type: object + description: The model interface. ml.create_connector: content: application/json: @@ -325,6 +384,16 @@ components: application/json: schema: $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' + ml.get_model@200: + content: + application/json: + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/Model' + ml.update_model@200: + content: + application/json: + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/UpdateModelResponse' ml.delete_task@200: content: application/json: @@ -385,6 +454,18 @@ components: required: true schema: type: string + ml.get_model::path.model_id: + name: model_id + in: path + required: true + schema: + type: string + ml.update_model::path.model_id: + name: model_id + in: path + required: true + schema: + type: string ml.deploy_model::path.model_id: name: model_id in: path diff --git a/spec/schemas/ml._common.yaml b/spec/schemas/ml._common.yaml index 1b83fcfce..3f1ac9963 100644 --- a/spec/schemas/ml._common.yaml +++ b/spec/schemas/ml._common.yaml @@ -225,6 +225,152 @@ components: - description - latest_version - name + Model: + type: object + properties: + name: + type: string + description: The model name. + model_group_id: + type: string + description: The model group ID. + algorithm: + type: string + description: The algorithm. + model_version: + type: string + description: The model version. + model_format: + type: string + description: The model format. + enum: + - ONNX + - TORCH_SCRIPT + model_state: + type: string + description: The model state. + enum: + - DEPLOYED + - DEPLOYING + - DEPLOY_FAILED + - PARTIALLY_DEPLOYED + - REGISTERED + - REGISTERING + model_content_size_in_bytes: + type: integer + format: int64 + description: The model content size in bytes. + model_content_hash_value: + type: string + description: The model content hash value. + model_config: + $ref: '#/components/schemas/ModelConfig' + created_time: + type: integer + format: int64 + description: The created time. + last_updated_time: + type: integer + format: int64 + description: The last updated time. + last_registered_time: + type: integer + format: int64 + description: The last registered time. + total_chunks: + type: integer + format: int64 + description: The total chunks. + is_hidden: + type: boolean + description: Whether the model is hidden. + required: + - model_state + RateLimiter: + type: object + properties: + limit: + type: integer + format: int64 + description: The maximum limit. + unit: + type: string + description: The unit of time. + enum: + - DAYS + - HOURS + - MICROSECONDS + - MILLISECONDS + - MINUTES + - NANOSECONDS + - SECONDS + required: + - limit + - unit + Guardrails: + type: object + properties: + type: + type: string + description: The guardrails type. + enum: + - local_regex + - model + input_guardrail: + $ref: '#/components/schemas/GuardrailsInputOutput' + output_guardrail: + $ref: '#/components/schemas/GuardrailsInputOutput' + stop_words: + $ref: '_common.analysis.yaml#/components/schemas/StopWords' + index_name: + $ref: '_common.yaml#/components/schemas/IndexName' + source_fields: + $ref: '_common.yaml#/components/schemas/Fields' + regex: + type: object + description: The regex used for input/output validation. + model_id: + type: string + description: The model ID. + response_filter: + type: string + description: The response filter. + response_validation_regex: + type: string + description: The response validation regex. + GuardrailsInputOutput: + type: object + properties: + model_id: + type: string + description: The model ID. + response_validation_regex: + type: string + description: The response validation regex. + UpdateModelResponse: + type: object + properties: + _index: + $ref: '_common.yaml#/components/schemas/IndexName' + _id: + $ref: '_common.yaml#/components/schemas/Id' + _version: + $ref: '_common.yaml#/components/schemas/VersionNumber' + result: + $ref: '_common.yaml#/components/schemas/Result' + forced_refresh: + type: boolean + description: Whether the model is forced to refresh. + _shards: + $ref: '_common.yaml#/components/schemas/ShardStatistics' + _seq_no: + $ref: '_common.yaml#/components/schemas/SequenceNumber' + _primary_term: + type: integer + format: int64 + description: The primary term. + required: + - _index Task: type: object properties: diff --git a/tests/plugins/ml/ml/models.yaml b/tests/plugins/ml/ml/models.yaml index 92cac3ca3..9d2b539e7 100644 --- a/tests/plugins/ml/ml/models.yaml +++ b/tests/plugins/ml/ml/models.yaml @@ -51,6 +51,37 @@ chapters: hits: hits: - _score: 1 + - synopsis: Search model. + path: /_plugins/_ml/models/_search + method: POST + request: + payload: + query: + match_all: {} + size: 1000 + response: + status: 200 + payload: + hits: + hits: + - _score: 1 + - synopsis: Get model. + path: /_plugins/_ml/models/{model_id} + method: GET + parameters: + model_id: ${get_completed_task.model_id} + response: + status: 200 + - synopsis: Update model. + path: /_plugins/_ml/models/{model_id} + method: PUT + parameters: + model_id: ${get_completed_task.model_id} + request: + payload: + name: updated_name + response: + status: 200 - synopsis: Delete model. path: /_plugins/_ml/models/{model_id} parameters: From a9712ff8cdb23c4d074a22f191b17eb09a0aa3d6 Mon Sep 17 00:00:00 2001 From: Nathalie Jonathan Date: Mon, 16 Dec 2024 17:45:38 -0800 Subject: [PATCH 03/14] Added ML Model APIs (undeploy, unload), moved test for search model to models/search.yaml, resolved conflicts and updated CHANGELOG Signed-off-by: Nathalie Jonathan --- CHANGELOG.md | 3 +- spec/namespaces/ml.yaml | 80 ++++++++++++++++++- spec/schemas/ml._common.yaml | 15 ++++ tests/plugins/ml/ml/models.yaml | 16 +--- tests/plugins/ml/models/search.yaml | 34 ++++++++ .../ml/models/undeploy_specific_models.yaml | 77 ++++++++++++++++++ tests/plugins/ml/models/unload.yaml | 74 +++++++++++++++++ .../ml/models/unload_specific_models.yaml | 76 ++++++++++++++++++ 8 files changed, 357 insertions(+), 18 deletions(-) create mode 100644 tests/plugins/ml/models/search.yaml create mode 100644 tests/plugins/ml/models/undeploy_specific_models.yaml create mode 100644 tests/plugins/ml/models/unload.yaml create mode 100644 tests/plugins/ml/models/unload_specific_models.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index be50b8000..4df039f3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added response schema for `PUT` and `DELETE /_plugins/_transform/{id}` ([#722](https://github.com/opensearch-project/opensearch-api-specification/pull/716)) - Added response schema for `GET /_plugins/_knn/warmup/{index}` ([#717](https://github.com/opensearch-project/opensearch-api-specification/pull/717)) - Added support for multiple test verbs ([#724](https://github.com/opensearch-project/opensearch-api-specification/pull/724)) -- Added ML Model APIs `GET /_plugins/_ml/models/{}`, `POST /_plugins/_ml/models/_search`, and `PUT /_plugins/_ml/models/{}` to the spec along with the tests. +- Added support for using a certificate and key in tests ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731)) +- Added `GET /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/_search`, `PUT /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/{model_id}/_unload`, `POST /_plugins/_ml/models/_unload`, and `POST /_plugins/_ml/models/_undeploy` ([#733](https://github.com/opensearch-project/opensearch-api-specification/pull/733)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index 1148c496e..aa576e787 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -93,6 +93,40 @@ paths: responses: '200': $ref: '#/components/responses/ml.undeploy_model@200' + /_plugins/_ml/models/_undeploy: + post: + operationId: ml.undeploy_model.1 + x-operation-group: ml.undeploy_model + description: Undeploys a model. + requestBody: + $ref: '#/components/requestBodies/ml.undeploy_model' + responses: + '200': + $ref: '#/components/responses/ml.undeploy_model@200' + /_plugins/_ml/models/{model_id}/_unload: + post: + operationId: ml.unload_model.0 + x-operation-group: ml.unload_model + deprecated: true + x-version-deprecated: 2.7.0 + x-deprecation-message: Use `undeploy_model` instead. + parameters: + - $ref: '#/components/parameters/ml.unload_model::path.model_id' + responses: + '200': + $ref: '#/components/responses/ml.unload_model@200' + /_plugins/_ml/models/_unload: + post: + operationId: ml.unload_model.1 + x-operation-group: ml.unload_model + deprecated: true + x-version-deprecated: 2.7.0 + x-deprecation-message: Use `undeploy_model` instead. + requestBody: + $ref: '#/components/requestBodies/ml.unload_model' + responses: + '200': + $ref: '#/components/responses/ml.unload_model@200' /_plugins/_ml/tasks/{task_id}: get: operationId: ml.get_task.0 @@ -123,7 +157,7 @@ paths: '200': $ref: '#/components/responses/ml.search_models@200' post: - operationId: ml.search_models.0 + operationId: ml.search_models.1 x-operation-group: ml.search_models description: Searches for models. requestBody: @@ -222,7 +256,6 @@ components: type: string description: The ID of the model group to which to register the model. required: - - model_format - name - version ml.search_models: @@ -271,6 +304,38 @@ components: interface: type: object description: The model interface. + ml.undeploy_model: + content: + application/json: + schema: + type: object + properties: + node_ids: + type: array + items: + $ref: '_common.yaml#/components/schemas/Id' + model_ids: + type: array + items: + $ref: '_common.yaml#/components/schemas/Id' + required: + - model_ids + ml.unload_model: + content: + application/json: + schema: + type: object + properties: + node_ids: + type: array + items: + $ref: '_common.yaml#/components/schemas/Id' + model_ids: + type: array + items: + $ref: '_common.yaml#/components/schemas/Id' + required: + - model_ids ml.create_connector: content: application/json: @@ -379,6 +444,11 @@ components: application/json: schema: $ref: '../schemas/ml._common.yaml#/components/schemas/UndeployModelResponse' + ml.unload_model@200: + content: + application/json: + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/UnloadModelResponse' ml.delete_model@200: content: application/json: @@ -478,6 +548,12 @@ components: required: true schema: type: string + ml.unload_model::path.model_id: + name: model_id + in: path + required: true + schema: + type: string ml.delete_task::path.task_id: name: task_id in: path diff --git a/spec/schemas/ml._common.yaml b/spec/schemas/ml._common.yaml index 3f1ac9963..de1b74492 100644 --- a/spec/schemas/ml._common.yaml +++ b/spec/schemas/ml._common.yaml @@ -115,6 +115,7 @@ components: - PARTIALLY_DEPLOYED - REGISTERED - REGISTERING + - UNDEPLOYED model_content_size_in_bytes: type: integer format: int64 @@ -256,6 +257,7 @@ components: - PARTIALLY_DEPLOYED - REGISTERED - REGISTERING + - UNDEPLOYED model_content_size_in_bytes: type: integer format: int64 @@ -424,6 +426,19 @@ components: UndeployModelNodeStats: type: object additionalProperties: true + UnloadModelResponse: + type: object + additionalProperties: + title: nodes + $ref: '#/components/schemas/UnloadModelNode' + UnloadModelNode: + type: object + properties: + stats: + $ref: '#/components/schemas/UnloadModelNodeStats' + UnloadModelNodeStats: + type: object + additionalProperties: true Credential: type: object properties: diff --git a/tests/plugins/ml/ml/models.yaml b/tests/plugins/ml/ml/models.yaml index 9d2b539e7..7258c5d07 100644 --- a/tests/plugins/ml/ml/models.yaml +++ b/tests/plugins/ml/ml/models.yaml @@ -51,20 +51,6 @@ chapters: hits: hits: - _score: 1 - - synopsis: Search model. - path: /_plugins/_ml/models/_search - method: POST - request: - payload: - query: - match_all: {} - size: 1000 - response: - status: 200 - payload: - hits: - hits: - - _score: 1 - synopsis: Get model. path: /_plugins/_ml/models/{model_id} method: GET @@ -79,7 +65,7 @@ chapters: model_id: ${get_completed_task.model_id} request: payload: - name: updated_name + name: updated response: status: 200 - synopsis: Delete model. diff --git a/tests/plugins/ml/models/search.yaml b/tests/plugins/ml/models/search.yaml new file mode 100644 index 000000000..98d79fa5f --- /dev/null +++ b/tests/plugins/ml/models/search.yaml @@ -0,0 +1,34 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the search of models. +distributions: + excluded: + - amazon-managed + - amazon-serverless +version: '>= 2.11' +prologues: + - path: /_plugins/_ml/models/_register + id: register_model + method: POST + request: + payload: + name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b + version: 1.0.1 + model_format: TORCH_SCRIPT + output: + task_id: payload.task_id +chapters: + - synopsis: Search model. + path: /_plugins/_ml/models/_search + method: POST + request: + payload: + query: + match_all: {} + size: 1000 + response: + status: 200 + payload: + hits: + hits: + - _score: 1 \ No newline at end of file diff --git a/tests/plugins/ml/models/undeploy_specific_models.yaml b/tests/plugins/ml/models/undeploy_specific_models.yaml new file mode 100644 index 000000000..77d43fe67 --- /dev/null +++ b/tests/plugins/ml/models/undeploy_specific_models.yaml @@ -0,0 +1,77 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the undeployment of specific models from specific nodes. +distributions: + excluded: + - amazon-managed + - amazon-serverless +version: '>= 2.11' +warnings: + multiple-paths-detected: false +prologues: + - path: /_plugins/_ml/models/_register + id: register_model + method: POST + request: + payload: + name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b + version: 1.0.1 + model_format: TORCH_SCRIPT + output: + task_id: payload.task_id +epilogues: + - path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${get_completed_register_model_task.model_id} + method: DELETE + status: [200, 404] +chapters: + - synopsis: Wait to get completed task. + id: get_completed_register_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${register_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Deploy a model. + id: deploy_model + path: /_plugins/_ml/models/{model_id}/_deploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + output: + task_id: payload.task_id + response: + status: 200 + - synopsis: Wait to get completed task. + id: get_completed_deploy_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${deploy_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Undeploy a model. + path: /_plugins/_ml/models/_undeploy + method: POST + request: + payload: + model_ids: + - ${get_completed_register_model_task.model_id} + response: + status: 200 diff --git a/tests/plugins/ml/models/unload.yaml b/tests/plugins/ml/models/unload.yaml new file mode 100644 index 000000000..4e6d6bcc7 --- /dev/null +++ b/tests/plugins/ml/models/unload.yaml @@ -0,0 +1,74 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the unload (deprecated) of a model from all ML nodes. +distributions: + excluded: + - amazon-managed + - amazon-serverless +warnings: + multiple-paths-detected: false +prologues: + - path: /_plugins/_ml/models/_register + id: register_model + method: POST + request: + payload: + name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b + version: 1.0.1 + model_format: TORCH_SCRIPT + output: + task_id: payload.task_id +epilogues: + - path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${get_completed_register_model_task.model_id} + method: DELETE + status: [200, 404] +chapters: + - synopsis: Wait to get completed task. + id: get_completed_register_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${register_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Deploy a model. + id: deploy_model + path: /_plugins/_ml/models/{model_id}/_deploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + output: + task_id: payload.task_id + response: + status: 200 + - synopsis: Wait to get completed task. + id: get_completed_deploy_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${deploy_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Unload a model. + path: /_plugins/_ml/models/{model_id}/_unload + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + response: + status: 200 \ No newline at end of file diff --git a/tests/plugins/ml/models/unload_specific_models.yaml b/tests/plugins/ml/models/unload_specific_models.yaml new file mode 100644 index 000000000..81bae92bf --- /dev/null +++ b/tests/plugins/ml/models/unload_specific_models.yaml @@ -0,0 +1,76 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the unload (deprecated) of specific models from specific nodes. +distributions: + excluded: + - amazon-managed + - amazon-serverless +warnings: + multiple-paths-detected: false +prologues: + - path: /_plugins/_ml/models/_register + id: register_model + method: POST + request: + payload: + name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b + version: 1.0.1 + model_format: TORCH_SCRIPT + output: + task_id: payload.task_id +epilogues: + - path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${get_completed_register_model_task.model_id} + method: DELETE + status: [200, 404] +chapters: + - synopsis: Wait to get completed task. + id: get_completed_register_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${register_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Deploy a model. + id: deploy_model + path: /_plugins/_ml/models/{model_id}/_deploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + output: + task_id: payload.task_id + response: + status: 200 + - synopsis: Wait to get completed task. + id: get_completed_deploy_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${deploy_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Unload a model. + path: /_plugins/_ml/models/_unload + method: POST + request: + payload: + model_ids: + - ${get_completed_register_model_task.model_id} + response: + status: 200 From d2710c70d8bd3e76377577d4e4b5b9f2c0dc1e3d Mon Sep 17 00:00:00 2001 From: Thomas Farr Date: Sat, 14 Dec 2024 01:54:54 +1300 Subject: [PATCH 04/14] Disambiguate snapshot schema naming (#730) Signed-off-by: Thomas Farr Signed-off-by: Nathalie Jonathan --- CHANGELOG.md | 1 + spec/namespaces/snapshot.yaml | 2 +- spec/schemas/snapshot._common.yaml | 24 ++++++++++++------------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4df039f3f..e66ce6d73 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Changed `tasks._common:TaskInfo` and `tasks._common:TaskGroup` to be composed of a `tasks._common:TaskInfoBase` ([#683](https://github.com/opensearch-project/opensearch-api-specification/pull/683)) - Changed `SearchModelsQuery`, `CreateConnectorRequest` & `RegisterAgentsRequest` to be defined inline of request bodies ([#725](https://github.com/opensearch-project/opensearch-api-specification/pull/725)) - Changed `indices.data_streams_stats:DataStreamsStatsItem` to instead be `indices._common:DataStreamStats` ([#725](https://github.com/opensearch-project/opensearch-api-specification/pull/725)) +- Changed naming of `snapshot._common`'s `Status`, `ShardsStats`, `ShardsStatsStage`, `ShardsStatsSummary` and `ShardsStatsSummaryItem` schemas to be prefixed with `Snapshot` ([#730](https://github.com/opensearch-project/opensearch-api-specification/pull/730)) ## [0.1.0] - 2024-10-25 diff --git a/spec/namespaces/snapshot.yaml b/spec/namespaces/snapshot.yaml index 1c4dc532c..f039049f2 100644 --- a/spec/namespaces/snapshot.yaml +++ b/spec/namespaces/snapshot.yaml @@ -478,7 +478,7 @@ components: snapshots: type: array items: - $ref: '../schemas/snapshot._common.yaml#/components/schemas/Status' + $ref: '../schemas/snapshot._common.yaml#/components/schemas/SnapshotStatus' required: - snapshots snapshot.verify_repository@200: diff --git a/spec/schemas/snapshot._common.yaml b/spec/schemas/snapshot._common.yaml index f894210dd..d21823669 100644 --- a/spec/schemas/snapshot._common.yaml +++ b/spec/schemas/snapshot._common.yaml @@ -104,7 +104,7 @@ components: $ref: '_common.yaml#/components/schemas/StringifiedBoolean' required: - location - Status: + SnapshotStatus: type: object properties: include_global_state: @@ -116,7 +116,7 @@ components: repository: type: string shards_stats: - $ref: '#/components/schemas/ShardsStats' + $ref: '#/components/schemas/SnapshotShardsStats' snapshot: type: string state: @@ -141,7 +141,7 @@ components: additionalProperties: $ref: '#/components/schemas/SnapshotShardsStatus' shards_stats: - $ref: '#/components/schemas/ShardsStats' + $ref: '#/components/schemas/SnapshotShardsStats' stats: $ref: '#/components/schemas/SnapshotStats' required: @@ -156,13 +156,13 @@ components: reason: type: string stage: - $ref: '#/components/schemas/ShardsStatsStage' + $ref: '#/components/schemas/SnapshotShardsStatsStage' stats: - $ref: '#/components/schemas/ShardsStatsSummary' + $ref: '#/components/schemas/SnapshotShardsStatsSummary' required: - stage - stats - ShardsStatsStage: + SnapshotShardsStatsStage: oneOf: - type: string const: DONE @@ -179,15 +179,15 @@ components: - type: string const: STARTED description: Number of shards in the snapshot that are in the started stage of being stored in the repository. - ShardsStatsSummary: + SnapshotShardsStatsSummary: type: object properties: incremental: - $ref: '#/components/schemas/ShardsStatsSummaryItem' + $ref: '#/components/schemas/SnapshotShardsStatsSummaryItem' processed: - $ref: '#/components/schemas/ShardsStatsSummaryItem' + $ref: '#/components/schemas/SnapshotShardsStatsSummaryItem' total: - $ref: '#/components/schemas/ShardsStatsSummaryItem' + $ref: '#/components/schemas/SnapshotShardsStatsSummaryItem' start_time_in_millis: $ref: '_common.yaml#/components/schemas/EpochTimeUnitMillis' time: @@ -199,7 +199,7 @@ components: - start_time_in_millis - time_in_millis - total - ShardsStatsSummaryItem: + SnapshotShardsStatsSummaryItem: type: object properties: file_count: @@ -210,7 +210,7 @@ components: required: - file_count - size_in_bytes - ShardsStats: + SnapshotShardsStats: type: object properties: done: From e2b287938e46f3f632ac22156bc1d398ecf82f51 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Mon, 16 Dec 2024 14:05:44 -0500 Subject: [PATCH 05/14] Added support for using a certificate and key. (#731) Signed-off-by: dblock Signed-off-by: Nathalie Jonathan --- .github/workflows/test-spec.yml | 6 ++ CHANGELOG.md | 3 +- DEVELOPER_GUIDE.md | 2 + TESTING_GUIDE.md | 14 ++++ spec/namespaces/security.yaml | 11 +++- spec/schemas/security._common.yaml | 5 ++ tests/plugins/security/.README.md | 1 + tests/plugins/security/.kirk-key.pem | 28 ++++++++ tests/plugins/security/.kirk.pem | 27 ++++++++ tests/plugins/security/api/nodesdn.yaml | 64 +++++++++++++++++++ tests/plugins/security/docker-compose.yml | 13 ++++ tools/src/OpenSearchHttpClient.ts | 19 +++++- .../dump-cluster-spec/dump-cluster-spec.ts | 4 ++ tools/src/tester/test.ts | 4 ++ .../tests/tester/OpenSearchHttpClient.test.ts | 43 +++++++++++++ tools/tests/tester/fixtures/keys/kirk-key.pem | 3 + tools/tests/tester/fixtures/keys/kirk.pem | 3 + 17 files changed, 246 insertions(+), 4 deletions(-) create mode 100644 tests/plugins/security/.README.md create mode 100644 tests/plugins/security/.kirk-key.pem create mode 100644 tests/plugins/security/.kirk.pem create mode 100644 tests/plugins/security/api/nodesdn.yaml create mode 100644 tests/plugins/security/docker-compose.yml create mode 100644 tools/tests/tester/fixtures/keys/kirk-key.pem create mode 100644 tools/tests/tester/fixtures/keys/kirk.pem diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml index c63dde722..d3b495ca1 100644 --- a/.github/workflows/test-spec.yml +++ b/.github/workflows/test-spec.yml @@ -53,6 +53,10 @@ jobs: tests: plugins/workload-management - version: 2.18.0 tests: plugins/analysis + - version: 2.18.0 + tests: plugins/security + cert: tests/plugins/security/.kirk.pem + key: tests/plugins/security/.kirk-key.pem - version: 2.19.0 hub: opensearchstaging ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc' @@ -97,6 +101,8 @@ jobs: --opensearch-version=${{ matrix.entry.version }} \ --coverage coverage/test-spec-coverage-${{ steps.tests.outputs.hash }}.json \ --opensearch-url=${{ matrix.entry.url || 'https://localhost:9200'}} \ + --opensearch-cert=${{ matrix.entry.cert }} \ + --opensearch-key=${{ matrix.entry.key }} \ --tests=tests/${{ matrix.entry.tests || 'default' }} - name: Get Container Logs diff --git a/CHANGELOG.md b/CHANGELOG.md index e66ce6d73..1507d6df2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,6 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added response schema for `GET /_plugins/_knn/warmup/{index}` ([#717](https://github.com/opensearch-project/opensearch-api-specification/pull/717)) - Added support for multiple test verbs ([#724](https://github.com/opensearch-project/opensearch-api-specification/pull/724)) - Added support for using a certificate and key in tests ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731)) -- Added `GET /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/_search`, `PUT /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/{model_id}/_unload`, `POST /_plugins/_ml/models/_unload`, and `POST /_plugins/_ml/models/_undeploy` ([#733](https://github.com/opensearch-project/opensearch-api-specification/pull/733)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) @@ -71,6 +70,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed response schema for `/_render/template` and `/_render/template/{id}` ([#724](https://github.com/opensearch-project/opensearch-api-specification/pull/724)) - Fixed data stream schema numeric property types ([#725](https://github.com/opensearch-project/opensearch-api-specification/pull/725)) - Fixed snapshot status numeric property types ([#729](https://github.com/opensearch-project/opensearch-api-specification/pull/729)) +- Fixed request schema for `PATCH /_plugins/_security/api/nodesdn` ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731)) +- Fixed response schema for `GET /_plugins/_security/api/nodesdn/{cluster_name}` ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731)) ### Changed - Changed `tasks._common:TaskInfo` and `tasks._common:TaskGroup` to be composed of a `tasks._common:TaskInfoBase` ([#683](https://github.com/opensearch-project/opensearch-api-specification/pull/683)) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 9709132fc..ffdc55b65 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -243,6 +243,8 @@ The dump-cluster-spec tool connects to an OpenSearch cluster which has the [open - `--opensearch-insecure`: Disable SSL/TLS certificate verification, defaults to performing verification. - `--opensearch-username `: The username to authenticate with the cluster, defaults to `admin`, only used when `--opensearch-password` is set. - `--opensearch-password `: The password to authenticate with the cluster, also settable via the `OPENSEARCH_PASSWORD` environment variable. +- `--opensearch-cert `: The OpenSSL certificate file, also settable via the `OPENSEARCH_CERT` environment variable. +- `--opensearch-key `: The OpenSSL certificate private key, also settable via the `OPENSEARCH_KEY` environment variable. - `--output `: The path to write the dumped spec to, defaults to `/build/opensearch-openapi-CLUSTER.yaml`. **Example** diff --git a/TESTING_GUIDE.md b/TESTING_GUIDE.md index 362f6359f..2812b37b2 100644 --- a/TESTING_GUIDE.md +++ b/TESTING_GUIDE.md @@ -5,6 +5,7 @@ - [Prerequisites](#prerequisites) - [OpenSearch Cluster](#opensearch-cluster) - [Run Tests](#run-tests) + - [Running Spec Tests that Require an Admin Certificate](#running-spec-tests-that-require-an-admin-certificate) - [Running Spec Tests with Amazon OpenSearch](#running-spec-tests-with-amazon-opensearch) - [Common Errors](#common-errors) - [401 Unauthorized](#401-unauthorized) @@ -76,6 +77,19 @@ Want to help with some missing tests? Choose from the remaining paths in the tes npm run test:spec -- --opensearch-insecure --coverage-report ``` +### Running Spec Tests that Require an Admin Certificate + +Some tests may require an admin certificate for authorization. The certificate can be provided wth `--opensearch-cert` and the key with `opensearch-key`. + +For example, run tests in [plugins/security](tests/plugins/security) as follows: +```bash +npm run test:spec--insecure -- \ + --tests tests/plugins/security/api/nodesdn.yaml \ + --opensearch-key tests/plugins/security/kirk-key.pem \ + --opensearch-cert tests/plugins/security/kirk.pem \ + --verbose +``` + ### Running Spec Tests with Amazon OpenSearch Use an Amazon OpenSearch service instance. diff --git a/spec/namespaces/security.yaml b/spec/namespaces/security.yaml index 2a63cd0d9..0a3a44bed 100644 --- a/spec/namespaces/security.yaml +++ b/spec/namespaces/security.yaml @@ -754,6 +754,8 @@ paths: responses: '200': $ref: '#/components/responses/security.update_distinguished_name@200' + '201': + $ref: '#/components/responses/security.update_distinguished_name@201' '400': $ref: '#/components/responses/security.update_distinguished_name@400' '403': @@ -1449,7 +1451,7 @@ components: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation' + $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperations' security.patch_distinguished_names: content: application/json: @@ -1843,7 +1845,7 @@ components: content: application/json: schema: - $ref: '../schemas/security._common.yaml#/components/schemas/DistinguishedNames' + $ref: '../schemas/security._common.yaml#/components/schemas/DistinguishedNamesMap' security.get_distinguished_name@400: content: application/json: @@ -2183,6 +2185,11 @@ components: application/json: schema: $ref: '../schemas/security._common.yaml#/components/schemas/Ok' + security.update_distinguished_name@201: + content: + application/json: + schema: + $ref: '../schemas/security._common.yaml#/components/schemas/Ok' security.update_distinguished_name@400: content: application/json: diff --git a/spec/schemas/security._common.yaml b/spec/schemas/security._common.yaml index c754e2325..ac53b2be7 100644 --- a/spec/schemas/security._common.yaml +++ b/spec/schemas/security._common.yaml @@ -524,6 +524,11 @@ components: type: string description: Message returned as part of CREATED response. + PatchOperations: + type: array + items: + $ref: '#/components/schemas/PatchOperation' + PatchOperation: type: object properties: diff --git a/tests/plugins/security/.README.md b/tests/plugins/security/.README.md new file mode 100644 index 000000000..239ff185d --- /dev/null +++ b/tests/plugins/security/.README.md @@ -0,0 +1 @@ +The demo certs in this directory do get updated from [the security repo](https://github.com/opensearch-project/security/tree/main/bwc-test/src/test/resources/security). The source of truth is [here](https://github.com/opensearch-project/security/blob/main/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java). diff --git a/tests/plugins/security/.kirk-key.pem b/tests/plugins/security/.kirk-key.pem new file mode 100644 index 000000000..fd1728cda --- /dev/null +++ b/tests/plugins/security/.kirk-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCVXDgEJQorgfXp +gpY0TgF55bD2xuzxN5Dc9rDfgWxrsOvOloMpd7k6FR71bKWjJi1KptSmM/cDElky +AWYKSfYWGiGxsQ+EQW+6kwCfEOHXQldn+0+JcWqP+osSPjtJfwRvRN5kRqP69MPo +7U0N2kdqenqMWjmG1chDGLRSOEGU5HIBiDxsZtOcvMaJ8b1eaW0lvS+6gFQ80AvB +GBkDDCOHHLtDXBylrZk2CQP8AzxNicIZ4B8G3CG3OHA8+nBtEtxZoIihrrkqlMt+ +b/5N8u8zB0Encew0kdrc4R/2wS//ahr6U+9Siq8T7WsUtGwKj3BJClg6OyDJRhlu +y2gFnxoPAgMBAAECggEAP5TOycDkx+megAWVoHV2fmgvgZXkBrlzQwUG/VZQi7V4 +ZGzBMBVltdqI38wc5MtbK3TCgHANnnKgor9iq02Z4wXDwytPIiti/ycV9CDRKvv0 +TnD2hllQFjN/IUh5n4thHWbRTxmdM7cfcNgX3aZGkYbLBVVhOMtn4VwyYu/Mxy8j +xClZT2xKOHkxqwmWPmdDTbAeZIbSv7RkIGfrKuQyUGUaWhrPslvYzFkYZ0umaDgQ +OAthZew5Bz3OfUGOMPLH61SVPuJZh9zN1hTWOvT65WFWfsPd2yStI+WD/5PU1Doo +1RyeHJO7s3ug8JPbtNJmaJwHe9nXBb/HXFdqb976yQKBgQDNYhpu+MYSYupaYqjs +9YFmHQNKpNZqgZ4ceRFZ6cMJoqpI5dpEMqToFH7tpor72Lturct2U9nc2WR0HeEs +/6tiptyMPTFEiMFb1opQlXF2ae7LeJllntDGN0Q6vxKnQV+7VMcXA0Y8F7tvGDy3 +qJu5lfvB1mNM2I6y/eMxjBuQhwKBgQC6K41DXMFro0UnoO879pOQYMydCErJRmjG +/tZSy3Wj4KA/QJsDSViwGfvdPuHZRaG9WtxdL6kn0w1exM9Rb0bBKl36lvi7o7xv +M+Lw9eyXMkww8/F5d7YYH77gIhGo+RITkKI3+5BxeBaUnrGvmHrpmpgRXWmINqr0 +0jsnN3u0OQKBgCf45vIgItSjQb8zonLz2SpZjTFy4XQ7I92gxnq8X0Q5z3B+o7tQ +K/4rNwTju/sGFHyXAJlX+nfcK4vZ4OBUJjP+C8CTjEotX4yTNbo3S6zjMyGQqDI5 +9aIOUY4pb+TzeUFJX7If5gR+DfGyQubvvtcg1K3GHu9u2l8FwLj87sRzAoGAflQF +RHuRiG+/AngTPnZAhc0Zq0kwLkpH2Rid6IrFZhGLy8AUL/O6aa0IGoaMDLpSWUJp +nBY2S57MSM11/MVslrEgGmYNnI4r1K25xlaqV6K6ztEJv6n69327MS4NG8L/gCU5 +3pEm38hkUi8pVYU7in7rx4TCkrq94OkzWJYurAkCgYATQCL/rJLQAlJIGulp8s6h +mQGwy8vIqMjAdHGLrCS35sVYBXG13knS52LJHvbVee39AbD5/LlWvjJGlQMzCLrw +F7oILW5kXxhb8S73GWcuMbuQMFVHFONbZAZgn+C9FW4l7XyRdkrbR1MRZ2km8YMs +/AHmo368d4PSNRMMzLHw8Q== +-----END PRIVATE KEY----- diff --git a/tests/plugins/security/.kirk.pem b/tests/plugins/security/.kirk.pem new file mode 100644 index 000000000..b89edfe18 --- /dev/null +++ b/tests/plugins/security/.kirk.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEmDCCA4CgAwIBAgIUaYSlET3nzsotWTrWueVPPh10yLcwDQYJKoZIhvcNAQEL +BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt +cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl +IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v +dCBDQTAeFw0yNDAyMjAxNzA0MjRaFw0zNDAyMTcxNzA0MjRaME0xCzAJBgNVBAYT +AmRlMQ0wCwYDVQQHDAR0ZXN0MQ8wDQYDVQQKDAZjbGllbnQxDzANBgNVBAsMBmNs +aWVudDENMAsGA1UEAwwEa2lyazCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAJVcOAQlCiuB9emCljROAXnlsPbG7PE3kNz2sN+BbGuw686Wgyl3uToVHvVs +paMmLUqm1KYz9wMSWTIBZgpJ9hYaIbGxD4RBb7qTAJ8Q4ddCV2f7T4lxao/6ixI+ +O0l/BG9E3mRGo/r0w+jtTQ3aR2p6eoxaOYbVyEMYtFI4QZTkcgGIPGxm05y8xonx +vV5pbSW9L7qAVDzQC8EYGQMMI4ccu0NcHKWtmTYJA/wDPE2JwhngHwbcIbc4cDz6 +cG0S3FmgiKGuuSqUy35v/k3y7zMHQSdx7DSR2tzhH/bBL/9qGvpT71KKrxPtaxS0 +bAqPcEkKWDo7IMlGGW7LaAWfGg8CAwEAAaOCASswggEnMAwGA1UdEwEB/wQCMAAw +DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMB0GA1UdDgQW +BBSjMS8tgguX/V7KSGLoGg7K6XMzIDCBzwYDVR0jBIHHMIHEgBQXh9+gWutmEqfV +0Pi6EkU8tysAnKGBlaSBkjCBjzETMBEGCgmSJomT8ixkARkWA2NvbTEXMBUGCgmS +JomT8ixkARkWB2V4YW1wbGUxGTAXBgNVBAoMEEV4YW1wbGUgQ29tIEluYy4xITAf +BgNVBAsMGEV4YW1wbGUgQ29tIEluYy4gUm9vdCBDQTEhMB8GA1UEAwwYRXhhbXBs +ZSBDb20gSW5jLiBSb290IENBghQNZAmZZn3EFOxBR4630XlhI+mo4jANBgkqhkiG +9w0BAQsFAAOCAQEACEUPPE66/Ot3vZqRGpjDjPHAdtOq+ebaglQhvYcnDw8LOZm8 +Gbh9M88CiO6UxC8ipQLTPh2yyeWArkpJzJK/Pi1eoF1XLiAa0sQ/RaJfQWPm9dvl +1ZQeK5vfD4147b3iBobwEV+CR04SKow0YeEEzAJvzr8YdKI6jqr+2GjjVqzxvRBy +KRVHWCFiR7bZhHGLq3br8hSu0hwjb3oGa1ZI8dui6ujyZt6nm6BoEkau3G/6+zq9 +E6vX3+8Fj4HKCAL6i0SwfGmEpTNp5WUhqibK/fMhhmMT4Mx6MxkT+OFnIjdUU0S/ +e3kgnG8qjficUr38CyEli1U0M7koIXUZI7r+LQ== +-----END CERTIFICATE----- diff --git a/tests/plugins/security/api/nodesdn.yaml b/tests/plugins/security/api/nodesdn.yaml new file mode 100644 index 000000000..176a748c9 --- /dev/null +++ b/tests/plugins/security/api/nodesdn.yaml @@ -0,0 +1,64 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test dynamic distinguished names. +epilogues: + - path: /_plugins/_security/api/nodesdn/opensearch-cluster + method: DELETE + status: [200,404] +chapters: + - synopsis: Add distinguished names. + path: /_plugins/_security/api/nodesdn/{cluster_name} + method: PUT + parameters: + cluster_name: opensearch-cluster + request: + payload: + nodes_dn: + - CN=cluster.example.com + response: + status: 201 + payload: + status: CREATED + - synopsis: Retrieve distinguished names. + path: /_plugins/_security/api/nodesdn + method: GET + - synopsis: Update distinguished names. + path: /_plugins/_security/api/nodesdn + method: PATCH + request: + payload: + - op: replace + path: /opensearch-cluster/nodes_dn/0 + value: CN=Christopher Nolan,CN=admin,DC=movies,DC=opensearch,DC=org + response: + status: 200 + payload: + status: OK + - synopsis: Update distinguished names (with cluster name). + path: /_plugins/_security/api/nodesdn/{cluster_name} + method: PATCH + parameters: + cluster_name: opensearch-cluster + request: + payload: + - op: replace + path: /nodes_dn/0 + value: CN=Quentin Tarantino,CN=admin,DC=movies,DC=opensearch,DC=org + response: + status: 200 + payload: + status: OK + - synopsis: Retrieve distinguished names (with cluster name). + path: /_plugins/_security/api/nodesdn/{cluster_name} + method: GET + parameters: + cluster_name: opensearch-cluster + - synopsis: Delete all distinguished names. + path: /_plugins/_security/api/nodesdn/{cluster_name} + method: DELETE + parameters: + cluster_name: opensearch-cluster + response: + status: 200 + payload: + status: OK diff --git a/tests/plugins/security/docker-compose.yml b/tests/plugins/security/docker-compose.yml new file mode 100644 index 000000000..f5050da5c --- /dev/null +++ b/tests/plugins/security/docker-compose.yml @@ -0,0 +1,13 @@ +version: '3' + +services: + opensearch-cluster: + image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF} + ports: + - 9200:9200 + - 9600:9600 + environment: + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!} + - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS} + - discovery.type=single-node + - plugins.security.nodes_dn_dynamic_config_enabled=true diff --git a/tools/src/OpenSearchHttpClient.ts b/tools/src/OpenSearchHttpClient.ts index 314a73d45..247f1f8f9 100644 --- a/tools/src/OpenSearchHttpClient.ts +++ b/tools/src/OpenSearchHttpClient.ts @@ -10,6 +10,7 @@ import { Option } from '@commander-js/extra-typings' import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse, type ResponseType } from 'axios' import * as https from 'node:https' +import fs from 'fs' import { sleep } from './helpers' import { Logger } from './Logger' import { aws4Interceptor } from 'aws4-axios' @@ -29,6 +30,12 @@ export const OPENSEARCH_USERNAME_OPTION = new Option('--opensearch-username ', 'password to use when authenticating with OpenSearch') .env('OPENSEARCH_PASSWORD') +export const OPENSEARCH_CERT_OPTION = new Option('--opensearch-cert ', 'client certificate file to use when authenticating with OpenSearch') + .env('OPENSEARCH_CERT') + +export const OPENSEARCH_KEY_OPTION = new Option('--opensearch-key ', 'client certificate private key file name to use when authenticating with OpenSearch') + .env('OPENSEARCH_KEY') + export const OPENSEARCH_INSECURE_OPTION = new Option('--opensearch-insecure', 'disable SSL/TLS certificate verification when connecting to OpenSearch') .default(DEFAULT_INSECURE) @@ -65,6 +72,8 @@ export interface AwsAuth { export interface OpenSearchHttpClientOptions { url?: string insecure?: boolean + cert?: string, + key?: string, responseType?: ResponseType logger?: Logger, basic_auth?: BasicAuth @@ -77,6 +86,8 @@ export type OpenSearchHttpClientCliOptions = { opensearchUsername?: string opensearchPassword?: string opensearchInsecure?: boolean + opensearchCert?: string, + opensearchKey?: string, awsAccessKeyId?: string awsSecretAccessKey?: string awsSessionToken?: string @@ -90,6 +101,8 @@ export function get_opensearch_opts_from_cli (opts: OpenSearchHttpClientCliOptio return { url: opts.opensearchUrl, insecure: opts.opensearchInsecure, + cert: opts.opensearchCert, + key: opts.opensearchKey, basic_auth: opts.opensearchUsername !== undefined && opts.opensearchPassword !== undefined ? { username: opts.opensearchUsername, password: opts.opensearchPassword @@ -164,7 +177,11 @@ export class OpenSearchHttpClient { this._axios = axios.create({ baseURL: opts?.url ?? DEFAULT_URL, - httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }), + httpsAgent: new https.Agent({ + rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE), + cert: opts?.cert !== undefined && opts?.cert !== '' ? fs.readFileSync(opts?.cert) : undefined, + key: opts?.key !== undefined && opts?.key !== '' ? fs.readFileSync(opts?.key) : undefined, + }), responseType: opts?.responseType, }) diff --git a/tools/src/dump-cluster-spec/dump-cluster-spec.ts b/tools/src/dump-cluster-spec/dump-cluster-spec.ts index 02dd77972..bdf8ea05a 100644 --- a/tools/src/dump-cluster-spec/dump-cluster-spec.ts +++ b/tools/src/dump-cluster-spec/dump-cluster-spec.ts @@ -13,7 +13,9 @@ import * as process from 'node:process' import { write_yaml } from '../helpers' import { get_opensearch_opts_from_cli, + OPENSEARCH_CERT_OPTION, OPENSEARCH_INSECURE_OPTION, + OPENSEARCH_KEY_OPTION, OPENSEARCH_PASSWORD_OPTION, OPENSEARCH_URL_OPTION, OPENSEARCH_USERNAME_OPTION, OpenSearchHttpClient, @@ -42,6 +44,8 @@ const command = new Command() .addOption(OPENSEARCH_USERNAME_OPTION) .addOption(OPENSEARCH_PASSWORD_OPTION) .addOption(OPENSEARCH_INSECURE_OPTION) + .addOption(OPENSEARCH_CERT_OPTION) + .addOption(OPENSEARCH_KEY_OPTION) .addOption(new Option('--output ', 'path to the output file').default(resolve(__dirname, '../../../build/opensearch-openapi-CLUSTER.yaml'))) .allowExcessArguments(false) .parse() diff --git a/tools/src/tester/test.ts b/tools/src/tester/test.ts index 77351fe75..8a69185fb 100644 --- a/tools/src/tester/test.ts +++ b/tools/src/tester/test.ts @@ -17,7 +17,9 @@ import { AWS_SERVICE_OPTION, AWS_SESSION_TOKEN_OPTION, get_opensearch_opts_from_cli, + OPENSEARCH_CERT_OPTION, OPENSEARCH_INSECURE_OPTION, + OPENSEARCH_KEY_OPTION, OPENSEARCH_PASSWORD_OPTION, OPENSEARCH_URL_OPTION, OPENSEARCH_USERNAME_OPTION, @@ -54,6 +56,8 @@ const command = new Command() .addOption(OPENSEARCH_USERNAME_OPTION) .addOption(OPENSEARCH_PASSWORD_OPTION) .addOption(OPENSEARCH_INSECURE_OPTION) + .addOption(OPENSEARCH_CERT_OPTION) + .addOption(OPENSEARCH_KEY_OPTION) .addOption(AWS_ACCESS_KEY_ID_OPTION) .addOption(AWS_SECRET_ACCESS_KEY_OPTION) .addOption(AWS_SESSION_TOKEN_OPTION) diff --git a/tools/tests/tester/OpenSearchHttpClient.test.ts b/tools/tests/tester/OpenSearchHttpClient.test.ts index 85c62af05..0493488c4 100644 --- a/tools/tests/tester/OpenSearchHttpClient.test.ts +++ b/tools/tests/tester/OpenSearchHttpClient.test.ts @@ -73,4 +73,47 @@ describe('OpenSearchHttpClient', () => { expect((await client.get('/')).data).toEqual({ called: true }) }) + + it('defaults to rejectUnauthorized', async () => { + let client = new OpenSearchHttpClient({ + url: 'https://localhost:9200' + }) + + mock.onAny().reply((config) => { + expect(config.httpsAgent.options.rejectUnauthorized).toBe(true) + return [200, { called: true }] + }) + + expect((await client.get('/')).data).toEqual({ called: true }) + }) + + it('sets rejectUnauthorized to false', async () => { + let client = new OpenSearchHttpClient({ + url: 'https://localhost:9200', + insecure: true + }) + + mock.onAny().reply((config) => { + expect(config.httpsAgent.options.rejectUnauthorized).toEqual(false) + return [200, { called: true }] + }) + + expect((await client.get('/')).data).toEqual({ called: true }) + }) + + it('adds a certificate file and key', async () => { + let client = new OpenSearchHttpClient({ + url: 'https://localhost:9200', + cert: './tools/tests/tester/fixtures/keys/kirk.pem', + key: './tools/tests/tester/fixtures/keys/kirk-key.pem' + }) + + mock.onAny().reply((config) => { + expect(config.httpsAgent.options.cert.toString()).toEqual("-----BEGIN CERTIFICATE-----\ncertificate\n-----END CERTIFICATE-----\n") + expect(config.httpsAgent.options.key.toString()).toEqual("-----BEGIN PRIVATE KEY-----\nprivate key\n-----END PRIVATE KEY-----\n") + return [200, { called: true }] + }) + + expect((await client.get('/')).data).toEqual({ called: true }) + }) }) diff --git a/tools/tests/tester/fixtures/keys/kirk-key.pem b/tools/tests/tester/fixtures/keys/kirk-key.pem new file mode 100644 index 000000000..5af21d7eb --- /dev/null +++ b/tools/tests/tester/fixtures/keys/kirk-key.pem @@ -0,0 +1,3 @@ +-----BEGIN PRIVATE KEY----- +private key +-----END PRIVATE KEY----- diff --git a/tools/tests/tester/fixtures/keys/kirk.pem b/tools/tests/tester/fixtures/keys/kirk.pem new file mode 100644 index 000000000..24c83bcd0 --- /dev/null +++ b/tools/tests/tester/fixtures/keys/kirk.pem @@ -0,0 +1,3 @@ +-----BEGIN CERTIFICATE----- +certificate +-----END CERTIFICATE----- From d989004fd591636717bc190e5fa159e73475df25 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 17 Dec 2024 08:10:14 -0500 Subject: [PATCH 06/14] Fixed /_search/scroll/{scroll_id}, missing search tests. (#732) * Fixed /_search/scroll. Signed-off-by: dblock * Added tests for GET and POST /_search. Signed-off-by: dblock * Added a test for GET /_search/pipeline and DELETE /_search/pipeline/{id}. Signed-off-by: dblock * Added missing _search/point_in_time tests. Signed-off-by: dblock --------- Signed-off-by: dblock Signed-off-by: Nathalie Jonathan --- CHANGELOG.md | 1 + spec/namespaces/_core.yaml | 3 - tests/default/_core/point_in_time/all.yaml | 33 ++++++ tests/default/_core/search.yaml | 19 ++++ tests/default/_core/search/pipeline.yaml | 33 ++++++ .../response_processor/rename_field.yaml | 13 +-- tests/default/_core/search/point_in_time.yaml | 37 ++++++ tests/default/_core/search/scroll.yaml | 107 ++++++++++++++++++ tests/default/indices/search.yaml | 38 +++++++ .../default/indices/search/point_in_time.yaml | 30 +++++ 10 files changed, 301 insertions(+), 13 deletions(-) create mode 100644 tests/default/_core/point_in_time/all.yaml create mode 100644 tests/default/_core/search.yaml create mode 100644 tests/default/_core/search/pipeline.yaml create mode 100644 tests/default/_core/search/point_in_time.yaml create mode 100644 tests/default/_core/search/scroll.yaml create mode 100644 tests/default/indices/search.yaml create mode 100644 tests/default/indices/search/point_in_time.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 1507d6df2..10b6dbbce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed snapshot status numeric property types ([#729](https://github.com/opensearch-project/opensearch-api-specification/pull/729)) - Fixed request schema for `PATCH /_plugins/_security/api/nodesdn` ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731)) - Fixed response schema for `GET /_plugins/_security/api/nodesdn/{cluster_name}` ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731)) +- Fixed request body for `/_search/scroll/{scroll_id}` ([#732](https://github.com/opensearch-project/opensearch-api-specification/pull/732)) ### Changed - Changed `tasks._common:TaskInfo` and `tasks._common:TaskGroup` to be composed of a `tasks._common:TaskInfoBase` ([#683](https://github.com/opensearch-project/opensearch-api-specification/pull/683)) diff --git a/spec/namespaces/_core.yaml b/spec/namespaces/_core.yaml index a1c0e68b0..229fc9e55 100644 --- a/spec/namespaces/_core.yaml +++ b/spec/namespaces/_core.yaml @@ -2523,9 +2523,6 @@ components: $ref: '../schemas/_common.yaml#/components/schemas/Duration' scroll_id: $ref: '../schemas/_common.yaml#/components/schemas/ScrollId' - required: - - scroll_id - description: The scroll ID if not passed by URL or query parameter. search: content: application/json: diff --git a/tests/default/_core/point_in_time/all.yaml b/tests/default/_core/point_in_time/all.yaml new file mode 100644 index 000000000..86ff71e03 --- /dev/null +++ b/tests/default/_core/point_in_time/all.yaml @@ -0,0 +1,33 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test point-in-time. +version: '>= 2.4' +epilogues: + - path: /_search/point_in_time/_all + method: DELETE + - path: /movies + method: DELETE + status: [200, 404] +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {title: The Cruise, year: 1998} + - {create: {_index: movies}} + - {title: Drive, year: 1960} + - path: /movies/_search/point_in_time + method: POST + parameters: + keep_alive: 1m +chapters: + - synopsis: Get all point in time. + path: /_search/point_in_time/_all + method: GET + - synopsis: Delete all point in time. + path: /_search/point_in_time/_all + method: DELETE diff --git a/tests/default/_core/search.yaml b/tests/default/_core/search.yaml new file mode 100644 index 000000000..88072e844 --- /dev/null +++ b/tests/default/_core/search.yaml @@ -0,0 +1,19 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test search. +chapters: + - synopsis: Search across all indexes (GET). + path: /_search + method: GET + request: + payload: + query: + match_all: {} + - synopsis: Search across all indexes (POST). + path: /_search + method: POST + request: + payload: + query: + match_all: {} + \ No newline at end of file diff --git a/tests/default/_core/search/pipeline.yaml b/tests/default/_core/search/pipeline.yaml new file mode 100644 index 000000000..eab34717f --- /dev/null +++ b/tests/default/_core/search/pipeline.yaml @@ -0,0 +1,33 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the creation of a search pipeline. +version: '>= 2.9' +epilogues: + - path: /_search/pipeline/empty-pipeline + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create search pipeline. + path: /_search/pipeline/{id} + method: PUT + parameters: + id: empty-pipeline + request: + payload: {} + response: + status: 200 + payload: + acknowledged: true + - synopsis: Query all pipelines. + path: /_search/pipeline + method: GET + - synopsis: Query created pipeline. + path: /_search/pipeline/{id} + method: GET + parameters: + id: empty-pipeline + - synopsis: Delete created pipeline. + path: /_search/pipeline/{id} + method: DELETE + parameters: + id: empty-pipeline diff --git a/tests/default/_core/search/pipeline/response_processor/rename_field.yaml b/tests/default/_core/search/pipeline/response_processor/rename_field.yaml index 388d1414a..d2b05957b 100644 --- a/tests/default/_core/search/pipeline/response_processor/rename_field.yaml +++ b/tests/default/_core/search/pipeline/response_processor/rename_field.yaml @@ -13,7 +13,7 @@ prologues: name: Drive status: [201] epilogues: - - path: /_search/pipeline/names_pipeline + - path: /_search/pipeline/rename-field method: DELETE status: [200, 404] - path: /movies @@ -24,7 +24,7 @@ chapters: path: /_search/pipeline/{id} method: PUT parameters: - id: names_pipeline + id: rename-field request: payload: response_processors: @@ -35,13 +35,6 @@ chapters: status: 200 payload: acknowledged: true - - synopsis: Query created pipeline. - path: /_search/pipeline/{id} - method: GET - parameters: - id: names_pipeline - response: - status: 200 - synopsis: Search. warnings: multiple-paths-detected: false @@ -49,7 +42,7 @@ chapters: method: GET parameters: index: movies - search_pipeline: names_pipeline + search_pipeline: rename-field response: status: 200 payload: diff --git a/tests/default/_core/search/point_in_time.yaml b/tests/default/_core/search/point_in_time.yaml new file mode 100644 index 000000000..7eba25818 --- /dev/null +++ b/tests/default/_core/search/point_in_time.yaml @@ -0,0 +1,37 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test point-in-time. +version: '>= 2.4' +epilogues: + - path: /_search/point_in_time/_all + method: DELETE + - path: /movies + method: DELETE + status: [200, 404] +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {title: The Cruise, year: 1998} + - {create: {_index: movies}} + - {title: Drive, year: 1960} + - path: /movies/_search/point_in_time + id: pit + method: POST + parameters: + keep_alive: 1m + output: + id: payload.pit_id +chapters: + - synopsis: Delete all pits. + path: /_search/point_in_time + request: + payload: + pit_id: + - ${pit.id} + method: DELETE diff --git a/tests/default/_core/search/scroll.yaml b/tests/default/_core/search/scroll.yaml new file mode 100644 index 000000000..fb4baccd1 --- /dev/null +++ b/tests/default/_core/search/scroll.yaml @@ -0,0 +1,107 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test search scroll. +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {title: The Lion King, year: 1994} + - {create: {_index: movies}} + - {title: Drive, year: 2011} + - {create: {_index: movies}} + - {title: Frozen, year: 2013} + - {create: {_index: movies}} + - {title: Moneyball, year: 2011} + - {create: {_index: movies}} + - {title: The Cruise, year: 1998} + - path: /movies/_search + method: GET + id: scroll + parameters: + scroll: 10m + request: + payload: + query: + match_all: {} + size: 1 + output: + scroll_id: payload._scroll_id +epilogues: + - path: /_search/scroll/_all + method: DELETE + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Get the next batch of results (GET). + method: GET + path: /_search/scroll + request: + payload: + scroll: 10m + scroll_id: ${scroll.scroll_id} + response: + status: 200 + payload: + hits: + hits: + - _index: movies + _source: + title: Drive + - synopsis: Get the next batch of results (POST). + method: POST + path: /_search/scroll + request: + payload: + scroll: 10m + scroll_id: ${scroll.scroll_id} + response: + status: 200 + payload: + hits: + hits: + - _index: movies + _source: + title: Frozen + - synopsis: Get the next batch of results (GET). + method: GET + path: /_search/scroll/{scroll_id} + parameters: + scroll_id: ${scroll.scroll_id} + request: + payload: + scroll: 10m + response: + status: 200 + payload: + hits: + hits: + - _index: movies + _source: + title: Moneyball + - synopsis: Get the next batch of results (POST). + method: POST + path: /_search/scroll/{scroll_id} + parameters: + scroll_id: ${scroll.scroll_id} + request: + payload: + scroll: 10m + response: + status: 200 + payload: + hits: + hits: + - _index: movies + _source: + title: The Cruise + - synopsis: Delete the scroll. + method: DELETE + path: /_search/scroll/{scroll_id} + parameters: + scroll_id: ${scroll.scroll_id} diff --git a/tests/default/indices/search.yaml b/tests/default/indices/search.yaml new file mode 100644 index 000000000..b3f78261b --- /dev/null +++ b/tests/default/indices/search.yaml @@ -0,0 +1,38 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test search. +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies, _id: movie1}} + - {director: Quentin Tarantino, title: Pulp Fiction, year: 1994} + - {create: {_index: movies, _id: movie2}} + - {director: Christopher Nolan, title: Inception, year: 2010} +epilogues: + - path: /movies + method: DELETE + status: [200, 404] +chapters: + - synopsis: Search in an index (GET). + path: /{index}/_search + method: GET + parameters: + index: movies + request: + payload: + query: + match_all: {} + - synopsis: Search across all indices (POST). + path: /{index}/_search + method: POST + parameters: + index: movies + request: + payload: + query: + match_all: {} diff --git a/tests/default/indices/search/point_in_time.yaml b/tests/default/indices/search/point_in_time.yaml new file mode 100644 index 000000000..c626db504 --- /dev/null +++ b/tests/default/indices/search/point_in_time.yaml @@ -0,0 +1,30 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test point-in-time. +version: '>= 2.4' +epilogues: + - path: /_search/point_in_time/_all + method: DELETE + - path: /movies + method: DELETE + status: [200, 404] +prologues: + - path: /_bulk + method: POST + parameters: + refresh: true + request: + content_type: application/x-ndjson + payload: + - {create: {_index: movies}} + - {title: The Cruise, year: 1998} + - {create: {_index: movies}} + - {title: Drive, year: 1960} +chapters: + - synopsis: Create a point in time. + path: /{index}/_search/point_in_time + method: POST + parameters: + index: + - movies + keep_alive: 1m From 58a24518ad7f178b466b88c1c3b35603a8c8a28b Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 17 Dec 2024 08:10:36 -0500 Subject: [PATCH 07/14] Added tests for /_template/{name}. (#736) Signed-off-by: dblock Signed-off-by: Nathalie Jonathan --- tests/default/_core/template.yaml | 73 +++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 tests/default/_core/template.yaml diff --git a/tests/default/_core/template.yaml b/tests/default/_core/template.yaml new file mode 100644 index 000000000..4096d9568 --- /dev/null +++ b/tests/default/_core/template.yaml @@ -0,0 +1,73 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Test templates. +epilogues: + - path: /_template/daily_logs + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create a template. + path: /_template/{name} + method: POST + parameters: + name: daily_logs + request: + payload: + index_patterns: + - 'logs*' + settings: + number_of_shards: 2 + number_of_replicas: 2 + response: + status: 200 + - synopsis: Update a template. + path: /_template/{name} + method: PUT + parameters: + name: daily_logs + request: + payload: + index_patterns: + - 'logs*' + aliases: + my_logs: {} + settings: + number_of_shards: 4 + number_of_replicas: 4 + mappings: + properties: + timestamp: + type: date + format: yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis + value: + type: double + response: + status: 200 + - synopsis: Retrieve templates. + path: /_template + method: GET + - synopsis: Retrieve a template. + path: /_template/{name} + method: GET + parameters: + name: daily_logs + response: + status: 200 + payload: + daily_logs: + index_patterns: + - logs* + settings: + index: + number_of_shards: '4' + number_of_replicas: '4' + - synopsis: Check that a template exists. + path: /_template/{name} + method: HEAD + parameters: + name: daily_logs + - synopsis: Delete a template. + path: /_template/{name} + method: DELETE + parameters: + name: daily_logs From 3b07554b52fe4eb9f522ef3b605dfa4e02d4d90a Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Tue, 17 Dec 2024 11:54:54 -0500 Subject: [PATCH 08/14] Added tests for /_validate/query. (#739) * Added tests for /_validate/query. Signed-off-by: dblock * Added retry for https://github.com/opensearch-project/opensearch-api-specification/issues/738. Signed-off-by: dblock --------- Signed-off-by: dblock Signed-off-by: Nathalie Jonathan --- tests/default/_core/validate/query.yaml | 17 +++++++++++++++++ tests/default/indices/validate/query.yaml | 23 +++++------------------ tests/plugins/security/api/nodesdn.yaml | 2 ++ 3 files changed, 24 insertions(+), 18 deletions(-) create mode 100644 tests/default/_core/validate/query.yaml diff --git a/tests/default/_core/validate/query.yaml b/tests/default/_core/validate/query.yaml new file mode 100644 index 000000000..1c5849c01 --- /dev/null +++ b/tests/default/_core/validate/query.yaml @@ -0,0 +1,17 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test validating queries. +chapters: + - synopsis: Validate a query. + path: /_validate/query + method: + - GET + - POST + request: + payload: + query: + match_all: {} + response: + status: 200 + payload: + valid: true diff --git a/tests/default/indices/validate/query.yaml b/tests/default/indices/validate/query.yaml index 264579058..55ced1fb8 100644 --- a/tests/default/indices/validate/query.yaml +++ b/tests/default/indices/validate/query.yaml @@ -1,6 +1,6 @@ $schema: ../../../../json_schemas/test_story.schema.yaml -description: Test validating queries for a specific index using both GET and POST methods. +description: Test validating queries for a specific index. epilogues: - path: /movies method: DELETE @@ -18,9 +18,11 @@ prologues: - {create: {_index: movies, _id: movie2}} - {director: Nicolas Winding Refn, title: Drive, year: 1960} chapters: - - synopsis: Validate a match query (GET). + - synopsis: Validate a match query. path: /{index}/_validate/query - method: GET + method: + - GET + - POST parameters: index: movies request: @@ -32,18 +34,3 @@ chapters: status: 200 payload: valid: true - - - synopsis: Validate a match query (POST). - path: /{index}/_validate/query - method: POST - parameters: - index: movies - request: - payload: - query: - match: - title: Drive - response: - status: 200 - payload: - valid: true \ No newline at end of file diff --git a/tests/plugins/security/api/nodesdn.yaml b/tests/plugins/security/api/nodesdn.yaml index 176a748c9..6a4611ef3 100644 --- a/tests/plugins/security/api/nodesdn.yaml +++ b/tests/plugins/security/api/nodesdn.yaml @@ -15,6 +15,8 @@ chapters: payload: nodes_dn: - CN=cluster.example.com + retry: + count: 3 response: status: 201 payload: From 55ad956037ffa671aea00a4a48796cb35b4e6d30 Mon Sep 17 00:00:00 2001 From: Nathalie Jonathan Date: Tue, 17 Dec 2024 09:41:42 -0800 Subject: [PATCH 09/14] Fixed lint and directory path error. Signed-off-by: Nathalie Jonathan --- spec/namespaces/ml.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index aa576e787..d65c5eb92 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -90,6 +90,8 @@ paths: description: Undeploys a model. parameters: - $ref: '#/components/parameters/ml.undeploy_model::path.model_id' + requestBody: + $ref: '#/components/requestBodies/ml.undeploy_model' responses: '200': $ref: '#/components/responses/ml.undeploy_model@200' @@ -110,8 +112,11 @@ paths: deprecated: true x-version-deprecated: 2.7.0 x-deprecation-message: Use `undeploy_model` instead. + description: Unloads a model. parameters: - $ref: '#/components/parameters/ml.unload_model::path.model_id' + requestBody: + $ref: '#/components/requestBodies/ml.unload_model' responses: '200': $ref: '#/components/responses/ml.unload_model@200' @@ -122,6 +127,7 @@ paths: deprecated: true x-version-deprecated: 2.7.0 x-deprecation-message: Use `undeploy_model` instead. + description: Unloads a model. requestBody: $ref: '#/components/requestBodies/ml.unload_model' responses: @@ -313,11 +319,11 @@ components: node_ids: type: array items: - $ref: '_common.yaml#/components/schemas/Id' + $ref: '../schemas/_common.yaml#/components/schemas/Id' model_ids: type: array items: - $ref: '_common.yaml#/components/schemas/Id' + $ref: '../schemas/_common.yaml#/components/schemas/Id' required: - model_ids ml.unload_model: @@ -329,11 +335,11 @@ components: node_ids: type: array items: - $ref: '_common.yaml#/components/schemas/Id' + $ref: '../schemas/_common.yaml#/components/schemas/Id' model_ids: type: array items: - $ref: '_common.yaml#/components/schemas/Id' + $ref: '../schemas/_common.yaml#/components/schemas/Id' required: - model_ids ml.create_connector: From 78cf49df5c943d41a4cdd7070e2745db6dfeca6a Mon Sep 17 00:00:00 2001 From: Nathalie Jonathan Date: Tue, 17 Dec 2024 14:42:53 -0800 Subject: [PATCH 10/14] Added ML Model APIs (load, upload, create metadata). Signed-off-by: Nathalie Jonathan --- CHANGELOG.md | 1 + spec/namespaces/ml.yaml | 159 ++++++++++++++++++++++++++ tests/plugins/ml/models/load.yaml | 79 +++++++++++++ tests/plugins/ml/models/metadata.yaml | 75 ++++++++++++ tests/plugins/ml/models/upload.yaml | 56 +++++++++ 5 files changed, 370 insertions(+) create mode 100644 tests/plugins/ml/models/load.yaml create mode 100644 tests/plugins/ml/models/metadata.yaml create mode 100644 tests/plugins/ml/models/upload.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 10b6dbbce..89f693614 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added response schema for `GET /_plugins/_knn/warmup/{index}` ([#717](https://github.com/opensearch-project/opensearch-api-specification/pull/717)) - Added support for multiple test verbs ([#724](https://github.com/opensearch-project/opensearch-api-specification/pull/724)) - Added support for using a certificate and key in tests ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731)) +- Added `GET /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/_search`, `PUT /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/{model_id}/_unload`, `POST /_plugins/_ml/models/_unload`, `POST /_plugins/_ml/models/_undeploy`, `POST /_plugins/_ml/models/_upload`,`POST /_plugins/_ml/models/{model_id}/_load`, and `POST /_plugins/_ml/models/meta` ([#733](https://github.com/opensearch-project/opensearch-api-specification/pull/733)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index d65c5eb92..8cecd7c88 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -43,6 +43,29 @@ paths: responses: '200': $ref: '#/components/responses/ml.register_model@200' + /_plugins/_ml/models/meta: + post: + operationId: ml.create_model_meta.0 + x-operation-group: ml.create_model_meta + description: Creates a model metadata. + requestBody: + $ref: '#/components/requestBodies/ml.create_model_meta' + responses: + '200': + $ref: '#/components/responses/ml.create_model_meta@200' + /_plugins/_ml/models/_upload: + post: + operationId: ml.upload_model.0 + x-operation-group: ml.upload_model + deprecated: true + x-version-deprecated: 2.7.0 + x-deprecation-message: Use `register_model` instead. + description: Registers a model. + requestBody: + $ref: '#/components/requestBodies/ml.upload_model' + responses: + '200': + $ref: '#/components/responses/ml.upload_model@200' /_plugins/_ml/models/{model_id}: get: operationId: ml.get_model.0 @@ -83,6 +106,19 @@ paths: responses: '200': $ref: '#/components/responses/ml.deploy_model@200' + /_plugins/_ml/models/{model_id}/_load: + post: + operationId: ml.load_model.0 + x-operation-group: ml.load_model + deprecated: true + x-version-deprecated: 2.7.0 + x-deprecation-message: Use `deploy_model` instead. + description: Deploys a model. + parameters: + - $ref: '#/components/parameters/ml.load_model::path.model_id' + responses: + '200': + $ref: '#/components/responses/ml.load_model@200' /_plugins/_ml/models/{model_id}/_undeploy: post: operationId: ml.undeploy_model.0 @@ -264,6 +300,72 @@ components: required: - name - version + ml.create_model_meta: + content: + application/json: + schema: + type: object + properties: + name: + type: string + description: The model name. + version: + type: string + description: The model version. + model_format: + type: string + description: The portable format of the model file. + enum: [ONNX, TORCH_SCRIPT] + model_group_id: + type: string + description: The ID of the model group to which to register the model. + model_content_hash_value: + type: string + description: The model content hash value. + model_config: + $ref: '../schemas/ml._common.yaml#/components/schemas/ModelConfig' + total_chunks: + type: integer + format: int64 + description: The total chunks. + url: + type: string + description: The model URL. + description: + type: string + description: The model description. + required: + - model_config + - model_content_hash_value + - model_format + - name + - total_chunks + - version + ml.upload_model: + content: + application/json: + schema: + type: object + properties: + name: + type: string + description: The model name. + version: + type: string + description: The model version. + model_format: + type: string + description: The portable format of the model file. + enum: [ONNX, TORCH_SCRIPT] + description: + type: string + description: The model description. + model_group_id: + type: string + description: The ID of the model group to which to register the model. + required: + - name + - version ml.search_models: content: application/json: @@ -430,6 +532,32 @@ components: required: - status - task_id + ml.create_model_meta@200: + content: + application/json: + schema: + type: object + properties: + status: + type: string + model_id: + type: string + required: + - model_id + - status + ml.upload_model@200: + content: + application/json: + schema: + type: object + properties: + status: + type: string + task_id: + type: string + required: + - status + - task_id ml.deploy_model@200: content: application/json: @@ -445,6 +573,31 @@ components: required: - status - task_id + ml.load_model@200: + content: + application/json: + schema: + type: object + properties: + status: + type: string + task_id: + type: string + task_type: + type: string + enum: + - BATCH_INGEST + - BATCH_PREDICTION + - DEPLOY_MODEL + - EXECUTION + - PREDICTION + - REGISTER_MODEL + - TRAINING + - TRAINING_AND_PREDICTION + required: + - status + - task_id + - task_type ml.undeploy_model@200: content: application/json: @@ -548,6 +701,12 @@ components: required: true schema: type: string + ml.load_model::path.model_id: + name: model_id + in: path + required: true + schema: + type: string ml.undeploy_model::path.model_id: name: model_id in: path diff --git a/tests/plugins/ml/models/load.yaml b/tests/plugins/ml/models/load.yaml new file mode 100644 index 000000000..c560e906c --- /dev/null +++ b/tests/plugins/ml/models/load.yaml @@ -0,0 +1,79 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the deployment of a model using load API (deprecated). +distributions: + excluded: + - amazon-managed + - amazon-serverless +warnings: + multiple-paths-detected: false +prologues: + - path: /_plugins/_ml/models/_register + id: register_model + method: POST + request: + payload: + name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b + version: 1.0.1 + model_format: TORCH_SCRIPT + output: + task_id: payload.task_id +epilogues: + - path: /_plugins/_ml/models/{model_id}/_undeploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + status: [200, 404] + - path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${get_completed_register_model_task.model_id} + method: DELETE + status: [200, 404] +chapters: + - synopsis: Wait to get completed task. + id: get_completed_register_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${register_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Deploy a model. + id: load_model + path: /_plugins/_ml/models/{model_id}/_load + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + output: + task_id: payload.task_id + response: + status: 200 + - synopsis: Wait to get completed task. + id: get_completed_load_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${load_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Undeploy a model. + path: /_plugins/_ml/models/{model_id}/_undeploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + response: + status: 200 diff --git a/tests/plugins/ml/models/metadata.yaml b/tests/plugins/ml/models/metadata.yaml new file mode 100644 index 000000000..49062cf61 --- /dev/null +++ b/tests/plugins/ml/models/metadata.yaml @@ -0,0 +1,75 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the creation of model metadata. +distributions: + excluded: + - amazon-managed + - amazon-serverless +version: '>= 2.11' +warnings: + multiple-paths-detected: false +prologues: + - path: /_plugins/_ml/models/_register + id: register_model + method: POST + request: + payload: + name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b + version: 1.0.1 + model_format: TORCH_SCRIPT + output: + task_id: payload.task_id +epilogues: + - path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${get_completed_task.model_id} + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create model metadata. + id: model_metadata + path: /_plugins/_ml/models/meta + method: POST + request: + payload: + name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b + version: 1.0.1 + model_format: TORCH_SCRIPT + model_content_hash_value: 123abc + total_chunks: 1 + model_config: + all_config: path + model_type: bert + embedding_dimension: 384 + framework_type: sentence_transformers + response: + status: 200 + output: + model_id: payload.model_id + - synopsis: Wait to get completed task. + id: get_completed_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${register_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 5 + wait: 30000 + - synopsis: Get model. + path: /_plugins/_ml/models/{model_id} + method: GET + parameters: + model_id: ${get_completed_task.model_id} + response: + status: 200 + - synopsis: Delete model. + path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${get_completed_task.model_id} + method: DELETE diff --git a/tests/plugins/ml/models/upload.yaml b/tests/plugins/ml/models/upload.yaml new file mode 100644 index 000000000..6efafdb48 --- /dev/null +++ b/tests/plugins/ml/models/upload.yaml @@ -0,0 +1,56 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the creation of models using upload API (deprecated). +distributions: + excluded: + - amazon-managed + - amazon-serverless +warnings: + multiple-paths-detected: false +epilogues: + - path: /_plugins/_ml/models/{model_id} + method: DELETE + status: [200, 404] + parameters: + model_id: ${get_completed_task.model_id} +chapters: + - synopsis: Register model. + id: upload_model + path: /_plugins/_ml/models/_upload + method: POST + request: + payload: + name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b + version: 1.0.1 + model_format: TORCH_SCRIPT + response: + status: 200 + output: + task_id: payload.task_id + - synopsis: Wait to get completed task. + id: get_completed_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${upload_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 5 + wait: 30000 + - synopsis: Get model. + path: /_plugins/_ml/models/{model_id} + method: GET + parameters: + model_id: ${get_completed_task.model_id} + response: + status: 200 + - synopsis: Delete model. + path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${get_completed_task.model_id} + method: DELETE From 9a5251683274eeb1f1fff9087ba9454c0e1af3cb Mon Sep 17 00:00:00 2001 From: Nathalie Jonathan Date: Wed, 18 Dec 2024 15:45:59 -0800 Subject: [PATCH 11/14] Added ML Model APIs (predict) and updated the test settings for metadata. Signed-off-by: Nathalie Jonathan --- CHANGELOG.md | 2 +- spec/namespaces/ml.yaml | 39 ++++++++ spec/schemas/ml._common.yaml | 62 +++++++++++++ tests/plugins/ml/models/metadata.yaml | 6 ++ tests/plugins/ml/models/predict.yaml | 124 ++++++++++++++++++++++++++ 5 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 tests/plugins/ml/models/predict.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 89f693614..ac808f014 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added response schema for `GET /_plugins/_knn/warmup/{index}` ([#717](https://github.com/opensearch-project/opensearch-api-specification/pull/717)) - Added support for multiple test verbs ([#724](https://github.com/opensearch-project/opensearch-api-specification/pull/724)) - Added support for using a certificate and key in tests ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731)) -- Added `GET /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/_search`, `PUT /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/{model_id}/_unload`, `POST /_plugins/_ml/models/_unload`, `POST /_plugins/_ml/models/_undeploy`, `POST /_plugins/_ml/models/_upload`,`POST /_plugins/_ml/models/{model_id}/_load`, and `POST /_plugins/_ml/models/meta` ([#733](https://github.com/opensearch-project/opensearch-api-specification/pull/733)) +- Added `GET /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/_search`, `PUT /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/{model_id}/_unload`, `POST /_plugins/_ml/models/_unload`, `POST /_plugins/_ml/models/_undeploy`, `POST /_plugins/_ml/models/_upload`,`POST /_plugins/_ml/models/{model_id}/_load`, `POST /_plugins/_ml/models/meta`, and `POST /_plugins/_ml/models/{model_id}/_predict` ([#733](https://github.com/opensearch-project/opensearch-api-specification/pull/733)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index 8cecd7c88..ec8790a66 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -169,6 +169,18 @@ paths: responses: '200': $ref: '#/components/responses/ml.unload_model@200' + /_plugins/_ml/models/{model_id}/_predict: + post: + operationId: ml.predict_model.0 + x-operation-group: ml.predict_model + description: Predicts a model. + parameters: + - $ref: '#/components/parameters/ml.predict_model::path.model_id' + requestBody: + $ref: '#/components/requestBodies/ml.predict_model' + responses: + '200': + $ref: '#/components/responses/ml.predict_model@200' /_plugins/_ml/tasks/{task_id}: get: operationId: ml.get_task.0 @@ -444,6 +456,22 @@ components: $ref: '../schemas/_common.yaml#/components/schemas/Id' required: - model_ids + ml.predict_model: + content: + application/json: + schema: + type: object + properties: + query_text: + type: string + description: The query text. + text_docs: + type: array + items: + type: string + description: The text documents. + required: + - text_docs ml.create_connector: content: application/json: @@ -608,6 +636,11 @@ components: application/json: schema: $ref: '../schemas/ml._common.yaml#/components/schemas/UnloadModelResponse' + ml.predict_model@200: + content: + application/json: + schema: + $ref: '../schemas/ml._common.yaml#/components/schemas/PredictModelResponse' ml.delete_model@200: content: application/json: @@ -719,6 +752,12 @@ components: required: true schema: type: string + ml.predict_model::path.model_id: + name: model_id + in: path + required: true + schema: + type: string ml.delete_task::path.task_id: name: task_id in: path diff --git a/spec/schemas/ml._common.yaml b/spec/schemas/ml._common.yaml index de1b74492..faa0da0c2 100644 --- a/spec/schemas/ml._common.yaml +++ b/spec/schemas/ml._common.yaml @@ -439,6 +439,68 @@ components: UnloadModelNodeStats: type: object additionalProperties: true + PredictModelResponse: + type: object + properties: + inference_results: + type: array + items: + $ref: '#/components/schemas/PredictModelResult' + PredictModelResult: + type: object + properties: + output: + type: array + items: + $ref: '#/components/schemas/PredictModelOutput' + PredictModelOutput: + type: object + properties: + name: + type: string + description: The output name. + data_type: + type: string + description: The output data type. + enum: + - BOOLEAN + - FLOAT16 + - FLOAT32 + - FLOAT64 + - INT32 + - INT64 + - INT8 + - STRING + - UINT8 + - UNKNOWN + shape: + type: array + items: + type: integer + format: int64 + description: The output shape. + data: + type: array + items: + type: number + format: double + description: The output data. + byte_buffer: + $ref: '#/components/schemas/ByteBuffer' + required: + - data + ByteBuffer: + type: object + properties: + array: + type: string + description: The byte buffer array. + order: + type: string + description: The byte buffer order. + enum: + - BIG_ENDIAN + - LITTLE_ENDIAN Credential: type: object properties: diff --git a/tests/plugins/ml/models/metadata.yaml b/tests/plugins/ml/models/metadata.yaml index 49062cf61..2b8753069 100644 --- a/tests/plugins/ml/models/metadata.yaml +++ b/tests/plugins/ml/models/metadata.yaml @@ -9,6 +9,12 @@ version: '>= 2.11' warnings: multiple-paths-detected: false prologues: + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + plugins.ml_commons.allow_registering_model_via_local_file: true - path: /_plugins/_ml/models/_register id: register_model method: POST diff --git a/tests/plugins/ml/models/predict.yaml b/tests/plugins/ml/models/predict.yaml new file mode 100644 index 000000000..d42080f70 --- /dev/null +++ b/tests/plugins/ml/models/predict.yaml @@ -0,0 +1,124 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the prediction of new data. +distributions: + excluded: + - amazon-managed + - amazon-serverless +warnings: + multiple-paths-detected: false +prologues: + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + plugins.ml_commons.only_run_on_ml_node: false + plugins.ml_commons.model_access_control_enabled: true + plugins.ml_commons.native_memory_threshold: 99 + - path: /_plugins/_ml/model_groups/_register + id: create_model_group + method: POST + request: + payload: + name: NLP_Group + description: Model group for NLP models. + output: + test_model_group_id: payload.model_group_id + - path: /_plugins/_ml/models/_register + id: register_model + method: POST + request: + payload: + name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b + version: 1.0.1 + model_format: TORCH_SCRIPT + output: + task_id: payload.task_id +epilogues: + - path: /_plugins/_ml/models/{model_id}/_undeploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + status: [200, 404] + - path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${get_completed_register_model_task.model_id} + method: DELETE + status: [200, 404] + - path: /_plugins/_ml/model_groups/{model_group_id} + method: DELETE + status: [200, 404] + parameters: + model_group_id: ${create_model_group.test_model_group_id} +chapters: + - synopsis: Wait to get completed task. + id: get_completed_register_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${register_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Deploy a model. + id: deploy_model + path: /_plugins/_ml/models/{model_id}/_deploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + output: + task_id: payload.task_id + response: + status: 200 + - synopsis: Wait to get completed task. + id: get_completed_deploy_model_task + path: /_plugins/_ml/tasks/{task_id} + method: GET + parameters: + task_id: ${deploy_model.task_id} + response: + status: 200 + payload: + state: COMPLETED + output: + model_id: payload.model_id + retry: + count: 3 + wait: 10000 + - synopsis: Predict new data. + id: predict_data + path: /_plugins/_ml/models/{model_id}/_predict + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + request: + payload: + query_text: today is sunny + text_docs: + - how are you + - it is winter + - today is july fifth + - today is sunny + response: + status: 200 + payload: + inference_results: + - output: + - data: + - 101 + output: + prediction_results: payload.inference_results[0].output[0].data + - synopsis: Undeploy a model. + path: /_plugins/_ml/models/{model_id}/_undeploy + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + response: + status: 200 \ No newline at end of file From 789ab69f863de6234772913882c368095f536803 Mon Sep 17 00:00:00 2001 From: Nathalie Jonathan Date: Thu, 19 Dec 2024 13:26:15 -0800 Subject: [PATCH 12/14] Fixed lint errors in predict.yaml, moved setup to prologues and teardown to epilogues in predict.yaml and load.yaml, updated CHANGELOG format, updated API description, 'model_group_id' ID type, 'version' parameter, and made 'model_format' a type of its own in ml.yaml. Signed-off-by: Nathalie Jonathan --- CHANGELOG.md | 4 +- spec/namespaces/ml.yaml | 34 ++++++----------- spec/schemas/_common.yaml | 7 +++- spec/schemas/ml._common.yaml | 9 +---- tests/plugins/ml/models/load.yaml | 37 +++++++----------- tests/plugins/ml/models/predict.yaml | 56 +++++++++++----------------- 6 files changed, 58 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff37c55b3..0d1ec8ca0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,11 +34,13 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added response schema for `GET /_plugins/_knn/warmup/{index}` ([#717](https://github.com/opensearch-project/opensearch-api-specification/pull/717)) - Added support for multiple test verbs ([#724](https://github.com/opensearch-project/opensearch-api-specification/pull/724)) - Added support for using a certificate and key in tests ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731)) -- Added `GET /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/_search`, `PUT /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/{model_id}/_unload`, `POST /_plugins/_ml/models/_unload`, `POST /_plugins/_ml/models/_undeploy`, `POST /_plugins/_ml/models/_upload`,`POST /_plugins/_ml/models/{model_id}/_load`, `POST /_plugins/_ml/models/meta`, and `POST /_plugins/_ml/models/{model_id}/_predict` ([#733](https://github.com/opensearch-project/opensearch-api-specification/pull/733)) - Added `_type` to `termvector` and `mtermvector` ([#734](https://github.com/opensearch-project/opensearch-api-specification/pull/734)) - Added missing `cancelled` and `resource_stats` to `/_reindex/{task_id}/_rethrottle` ([#740](https://github.com/opensearch-project/opensearch-api-specification/pull/740)) - Added missing `cancellation_time_millis` to `POST /_tasks/_cancel` ([#747](https://github.com/opensearch-project/opensearch-api-specification/pull/747)) - Added support for combining output variables ([#737](https://github.com/opensearch-project/opensearch-api-specification/pull/737)) +- Added 404 response to `/_search/scroll` ([#749](https://github.com/opensearch-project/opensearch-api-specification/pull/749)) +- Added `node_failures` to `DELETE /_search/scroll` and `DELETE /_search/scroll/{scroll_id}` ([#749](https://github.com/opensearch-project/opensearch-api-specification/pull/749)) +- Added `GET /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/_search`, `POST /_plugins/_ml/models/_unload`, `_undeploy`, `_upload`, `meta`, `POST /_plugins/_ml/models/{model_id}/_load`, `_predict`, `_unload`, and `PUT /_plugins/_ml/models/{model_id}` ([#733](https://github.com/opensearch-project/opensearch-api-specification/pull/733)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index ec8790a66..6304c893e 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -47,7 +47,7 @@ paths: post: operationId: ml.create_model_meta.0 x-operation-group: ml.create_model_meta - description: Creates a model metadata. + description: Creates model metadata. requestBody: $ref: '#/components/requestBodies/ml.create_model_meta' responses: @@ -297,18 +297,14 @@ components: type: string description: The model name. version: - type: string - description: The model version. + $ref: '../schemas/_common.yaml#/components/schemas/VersionString' model_format: - type: string - description: The portable format of the model file. - enum: [ONNX, TORCH_SCRIPT] + $ref: '../schemas/_common.yaml#/components/schemas/ModelFormat' description: type: string description: The model description. model_group_id: - type: string - description: The ID of the model group to which to register the model. + $ref: '../schemas/_common.yaml#/components/schemas/Id' required: - name - version @@ -322,15 +318,11 @@ components: type: string description: The model name. version: - type: string - description: The model version. + $ref: '../schemas/_common.yaml#/components/schemas/VersionString' model_format: - type: string - description: The portable format of the model file. - enum: [ONNX, TORCH_SCRIPT] + $ref: '../schemas/_common.yaml#/components/schemas/ModelFormat' model_group_id: - type: string - description: The ID of the model group to which to register the model. + $ref: '../schemas/_common.yaml#/components/schemas/Id' model_content_hash_value: type: string description: The model content hash value. @@ -339,7 +331,7 @@ components: total_chunks: type: integer format: int64 - description: The total chunks. + description: Number of chunks the model is split into. url: type: string description: The model URL. @@ -363,18 +355,14 @@ components: type: string description: The model name. version: - type: string - description: The model version. + $ref: '../schemas/_common.yaml#/components/schemas/VersionString' model_format: - type: string - description: The portable format of the model file. - enum: [ONNX, TORCH_SCRIPT] + $ref: '../schemas/_common.yaml#/components/schemas/ModelFormat' description: type: string description: The model description. model_group_id: - type: string - description: The ID of the model group to which to register the model. + $ref: '../schemas/_common.yaml#/components/schemas/Id' required: - name - version diff --git a/spec/schemas/_common.yaml b/spec/schemas/_common.yaml index f5ab3b899..7057e1736 100644 --- a/spec/schemas/_common.yaml +++ b/spec/schemas/_common.yaml @@ -2380,4 +2380,9 @@ components: type: number required: - active_threads - - thread_executions \ No newline at end of file + - thread_executions + ModelFormat: + type: string + enum: + - ONNX + - TORCH_SCRIPT \ No newline at end of file diff --git a/spec/schemas/ml._common.yaml b/spec/schemas/ml._common.yaml index faa0da0c2..08a781809 100644 --- a/spec/schemas/ml._common.yaml +++ b/spec/schemas/ml._common.yaml @@ -129,8 +129,7 @@ components: model_config: $ref: '#/components/schemas/ModelConfig' model_format: - type: string - description: The model format. + $ref: '_common.yaml#/components/schemas/ModelFormat' model_task_type: type: string description: The model task type. @@ -242,11 +241,7 @@ components: type: string description: The model version. model_format: - type: string - description: The model format. - enum: - - ONNX - - TORCH_SCRIPT + $ref: '_common.yaml#/components/schemas/ModelFormat' model_state: type: string description: The model state. diff --git a/tests/plugins/ml/models/load.yaml b/tests/plugins/ml/models/load.yaml index c560e906c..8f2b69b8f 100644 --- a/tests/plugins/ml/models/load.yaml +++ b/tests/plugins/ml/models/load.yaml @@ -18,6 +18,19 @@ prologues: model_format: TORCH_SCRIPT output: task_id: payload.task_id + - path: /_plugins/_ml/tasks/{task_id} + id: get_completed_register_model_task + method: GET + parameters: + task_id: ${register_model.task_id} + retry: + count: 3 + wait: 10000 + until: + - path: payload.state + equal: COMPLETED + output: + model_id: payload.model_id epilogues: - path: /_plugins/_ml/models/{model_id}/_undeploy method: POST @@ -30,21 +43,6 @@ epilogues: method: DELETE status: [200, 404] chapters: - - synopsis: Wait to get completed task. - id: get_completed_register_model_task - path: /_plugins/_ml/tasks/{task_id} - method: GET - parameters: - task_id: ${register_model.task_id} - response: - status: 200 - payload: - state: COMPLETED - output: - model_id: payload.model_id - retry: - count: 3 - wait: 10000 - synopsis: Deploy a model. id: load_model path: /_plugins/_ml/models/{model_id}/_load @@ -69,11 +67,4 @@ chapters: model_id: payload.model_id retry: count: 3 - wait: 10000 - - synopsis: Undeploy a model. - path: /_plugins/_ml/models/{model_id}/_undeploy - method: POST - parameters: - model_id: ${get_completed_register_model_task.model_id} - response: - status: 200 + wait: 10000 diff --git a/tests/plugins/ml/models/predict.yaml b/tests/plugins/ml/models/predict.yaml index d42080f70..f2674e0c9 100644 --- a/tests/plugins/ml/models/predict.yaml +++ b/tests/plugins/ml/models/predict.yaml @@ -35,6 +35,26 @@ prologues: model_format: TORCH_SCRIPT output: task_id: payload.task_id + - path: /_plugins/_ml/tasks/{task_id} + id: get_completed_register_model_task + method: GET + parameters: + task_id: ${register_model.task_id} + retry: + count: 3 + wait: 10000 + until: + - path: payload.state + equal: COMPLETED + output: + model_id: payload.model_id + - path: /_plugins/_ml/models/{model_id}/_deploy + id: deploy_model + method: POST + parameters: + model_id: ${get_completed_register_model_task.model_id} + output: + task_id: payload.task_id epilogues: - path: /_plugins/_ml/models/{model_id}/_undeploy method: POST @@ -52,31 +72,6 @@ epilogues: parameters: model_group_id: ${create_model_group.test_model_group_id} chapters: - - synopsis: Wait to get completed task. - id: get_completed_register_model_task - path: /_plugins/_ml/tasks/{task_id} - method: GET - parameters: - task_id: ${register_model.task_id} - response: - status: 200 - payload: - state: COMPLETED - output: - model_id: payload.model_id - retry: - count: 3 - wait: 10000 - - synopsis: Deploy a model. - id: deploy_model - path: /_plugins/_ml/models/{model_id}/_deploy - method: POST - parameters: - model_id: ${get_completed_register_model_task.model_id} - output: - task_id: payload.task_id - response: - status: 200 - synopsis: Wait to get completed task. id: get_completed_deploy_model_task path: /_plugins/_ml/tasks/{task_id} @@ -111,14 +106,7 @@ chapters: payload: inference_results: - output: - - data: - - 101 + - data: + - 101 output: prediction_results: payload.inference_results[0].output[0].data - - synopsis: Undeploy a model. - path: /_plugins/_ml/models/{model_id}/_undeploy - method: POST - parameters: - model_id: ${get_completed_register_model_task.model_id} - response: - status: 200 \ No newline at end of file From 382477b38dd89e2276ba11a2ade75320eb972fc9 Mon Sep 17 00:00:00 2001 From: Nathalie Jonathan Date: Thu, 19 Dec 2024 14:35:53 -0800 Subject: [PATCH 13/14] Added ML Model APIs (chunk, register meta), updated CHANGELOG and test for deprecated model metadata creation API. Signed-off-by: Nathalie Jonathan --- CHANGELOG.md | 2 +- spec/namespaces/ml.yaml | 112 +++++++++++++++++- tests/plugins/ml/models/create_metadata.yaml | 42 +++++++ .../{metadata.yaml => register_metadata.yaml} | 70 +++++------ 4 files changed, 190 insertions(+), 36 deletions(-) create mode 100644 tests/plugins/ml/models/create_metadata.yaml rename tests/plugins/ml/models/{metadata.yaml => register_metadata.yaml} (53%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d1ec8ca0..5f1431347 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,7 +40,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added support for combining output variables ([#737](https://github.com/opensearch-project/opensearch-api-specification/pull/737)) - Added 404 response to `/_search/scroll` ([#749](https://github.com/opensearch-project/opensearch-api-specification/pull/749)) - Added `node_failures` to `DELETE /_search/scroll` and `DELETE /_search/scroll/{scroll_id}` ([#749](https://github.com/opensearch-project/opensearch-api-specification/pull/749)) -- Added `GET /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/_search`, `POST /_plugins/_ml/models/_unload`, `_undeploy`, `_upload`, `meta`, `POST /_plugins/_ml/models/{model_id}/_load`, `_predict`, `_unload`, and `PUT /_plugins/_ml/models/{model_id}` ([#733](https://github.com/opensearch-project/opensearch-api-specification/pull/733)) +- Added `GET /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/_search`, `POST /_plugins/_ml/models/_unload`, `_undeploy`, `_upload`, `meta`, `_register_meta`, `POST /_plugins/_ml/models/{model_id}/_load`, `_predict`, `_unload`, `chunk/{index}`, and `PUT /_plugins/_ml/models/{model_id}` ([#733](https://github.com/opensearch-project/opensearch-api-specification/pull/733)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index 6304c893e..8ab6a2a0c 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -47,12 +47,37 @@ paths: post: operationId: ml.create_model_meta.0 x-operation-group: ml.create_model_meta - description: Creates model metadata. + deprecated: true + x-deprecation-message: Use `_register_meta` instead. + description: Registers model metadata. requestBody: $ref: '#/components/requestBodies/ml.create_model_meta' responses: '200': $ref: '#/components/responses/ml.create_model_meta@200' + /_plugins/_ml/models/_register_meta: + post: + operationId: ml.register_model_meta.0 + x-operation-group: ml.register_model_meta + description: Registers model metadata. + requestBody: + $ref: '#/components/requestBodies/ml.register_model_meta' + responses: + '200': + $ref: '#/components/responses/ml.register_model_meta@200' + /_plugins/_ml/models/{model_id}/chunk/{index}: + post: + operationId: ml.chunk_model.0 + x-operation-group: ml.chunk_model + description: Chunks model. + parameters: + - $ref: '#/components/parameters/ml.chunk_model::path.index' + - $ref: '#/components/parameters/ml.chunk_model::path.model_id' + requestBody: + $ref: '#/components/requestBodies/ml.chunk_model' + responses: + '200': + $ref: '#/components/responses/ml.chunk_model@200' /_plugins/_ml/models/_upload: post: operationId: ml.upload_model.0 @@ -345,6 +370,54 @@ components: - name - total_chunks - version + ml.register_model_meta: + content: + application/json: + schema: + type: object + properties: + name: + type: string + description: The model name. + version: + $ref: '../schemas/_common.yaml#/components/schemas/VersionString' + model_format: + $ref: '../schemas/_common.yaml#/components/schemas/ModelFormat' + model_group_id: + $ref: '../schemas/_common.yaml#/components/schemas/Id' + model_content_hash_value: + type: string + description: The model content hash value. + model_config: + $ref: '../schemas/ml._common.yaml#/components/schemas/ModelConfig' + total_chunks: + type: integer + format: int64 + description: Number of chunks the model is split into. + url: + type: string + description: The model URL. + description: + type: string + description: The model description. + required: + - model_config + - model_content_hash_value + - model_format + - name + - total_chunks + - version + ml.chunk_model: + content: + application/json: + schema: + type: object + properties: + chunk: + type: string + description: The model chunk. + required: + - chunk ml.upload_model: content: application/json: @@ -561,6 +634,30 @@ components: required: - model_id - status + ml.register_model_meta@200: + content: + application/json: + schema: + type: object + properties: + status: + type: string + model_id: + type: string + required: + - model_id + - status + ml.chunk_model@200: + content: + application/json: + schema: + type: object + properties: + status: + type: string + description: The status of the chunking operation. + required: + - status ml.upload_model@200: content: application/json: @@ -746,6 +843,19 @@ components: required: true schema: type: string + ml.chunk_model::path.index: + name: index + in: path + required: true + schema: + type: integer + format: int64 + ml.chunk_model::path.model_id: + name: model_id + in: path + required: true + schema: + type: string ml.delete_task::path.task_id: name: task_id in: path diff --git a/tests/plugins/ml/models/create_metadata.yaml b/tests/plugins/ml/models/create_metadata.yaml new file mode 100644 index 000000000..4ad59ccd7 --- /dev/null +++ b/tests/plugins/ml/models/create_metadata.yaml @@ -0,0 +1,42 @@ +$schema: ../../../../json_schemas/test_story.schema.yaml + +description: Test the creation of model metadata using meta API (deprecated). +distributions: + excluded: + - amazon-managed + - amazon-serverless +version: '>= 2.11' +prologues: + - path: /_cluster/settings + method: PUT + request: + payload: + persistent: + plugins.ml_commons.allow_registering_model_via_local_file: true +epilogues: + - path: /_plugins/_ml/models/{model_id} + parameters: + model_id: ${model_metadata.model_id} + method: DELETE + status: [200, 404] +chapters: + - synopsis: Create model metadata. + id: model_metadata + path: /_plugins/_ml/models/meta + method: POST + request: + payload: + name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b + version: 1.0.1 + model_format: TORCH_SCRIPT + model_content_hash_value: 123abc + total_chunks: 3 + model_config: + all_config: path + model_type: bert + embedding_dimension: 384 + framework_type: sentence_transformers + response: + status: 200 + output: + model_id: payload.model_id \ No newline at end of file diff --git a/tests/plugins/ml/models/metadata.yaml b/tests/plugins/ml/models/register_metadata.yaml similarity index 53% rename from tests/plugins/ml/models/metadata.yaml rename to tests/plugins/ml/models/register_metadata.yaml index 2b8753069..30faf08a1 100644 --- a/tests/plugins/ml/models/metadata.yaml +++ b/tests/plugins/ml/models/register_metadata.yaml @@ -1,6 +1,6 @@ $schema: ../../../../json_schemas/test_story.schema.yaml -description: Test the creation of model metadata. +description: Test the creation of model metadata and chunking. distributions: excluded: - amazon-managed @@ -15,26 +15,16 @@ prologues: payload: persistent: plugins.ml_commons.allow_registering_model_via_local_file: true - - path: /_plugins/_ml/models/_register - id: register_model - method: POST - request: - payload: - name: huggingface/sentence-transformers/msmarco-distilbert-base-tas-b - version: 1.0.1 - model_format: TORCH_SCRIPT - output: - task_id: payload.task_id epilogues: - path: /_plugins/_ml/models/{model_id} parameters: - model_id: ${get_completed_task.model_id} + model_id: ${model_metadata.model_id} method: DELETE status: [200, 404] chapters: - - synopsis: Create model metadata. + - synopsis: Register model metadata. id: model_metadata - path: /_plugins/_ml/models/meta + path: /_plugins/_ml/models/_register_meta method: POST request: payload: @@ -42,7 +32,7 @@ chapters: version: 1.0.1 model_format: TORCH_SCRIPT model_content_hash_value: 123abc - total_chunks: 1 + total_chunks: 3 model_config: all_config: path model_type: bert @@ -52,30 +42,42 @@ chapters: status: 200 output: model_id: payload.model_id - - synopsis: Wait to get completed task. - id: get_completed_task - path: /_plugins/_ml/tasks/{task_id} - method: GET + - synopsis: Upload chunk 0. + path: /_plugins/_ml/models/{model_id}/chunk/{index} + method: POST parameters: - task_id: ${register_model.task_id} + index: 0 + model_id: ${model_metadata.model_id} + request: + payload: + chunk: chunk0 response: status: 200 payload: - state: COMPLETED - output: - model_id: payload.model_id - retry: - count: 5 - wait: 30000 - - synopsis: Get model. - path: /_plugins/_ml/models/{model_id} - method: GET + status: Uploaded + - synopsis: Upload chunk 1. + path: /_plugins/_ml/models/{model_id}/chunk/{index} + method: POST parameters: - model_id: ${get_completed_task.model_id} + index: 1 + model_id: ${model_metadata.model_id} + request: + payload: + chunk: chunk1 response: status: 200 - - synopsis: Delete model. - path: /_plugins/_ml/models/{model_id} + payload: + status: Uploaded + - synopsis: Upload chunk 2. + path: /_plugins/_ml/models/{model_id}/chunk/{index} + method: POST parameters: - model_id: ${get_completed_task.model_id} - method: DELETE + index: 2 + model_id: ${model_metadata.model_id} + request: + payload: + chunk: chunk2 + response: + status: 200 + payload: + status: Uploaded From 33aab0291a88c6a168aaf99a9e242854e0a239ae Mon Sep 17 00:00:00 2001 From: Nathalie Jonathan Date: Fri, 20 Dec 2024 11:54:16 -0800 Subject: [PATCH 14/14] Added ML Model APIs (upload chunk), moved test for deprecated chunk upload to create_metadata.yaml, updated CHANGELOG. Signed-off-by: Nathalie Jonathan --- CHANGELOG.md | 3 +- spec/namespaces/ml.yaml | 62 +++++++++++++++++-- tests/plugins/ml/models/create_metadata.yaml | 46 +++++++++++++- .../plugins/ml/models/register_metadata.yaml | 14 ++--- 4 files changed, 108 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f1431347..d971a52e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,7 +40,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added support for combining output variables ([#737](https://github.com/opensearch-project/opensearch-api-specification/pull/737)) - Added 404 response to `/_search/scroll` ([#749](https://github.com/opensearch-project/opensearch-api-specification/pull/749)) - Added `node_failures` to `DELETE /_search/scroll` and `DELETE /_search/scroll/{scroll_id}` ([#749](https://github.com/opensearch-project/opensearch-api-specification/pull/749)) -- Added `GET /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/_search`, `POST /_plugins/_ml/models/_unload`, `_undeploy`, `_upload`, `meta`, `_register_meta`, `POST /_plugins/_ml/models/{model_id}/_load`, `_predict`, `_unload`, `chunk/{index}`, and `PUT /_plugins/_ml/models/{model_id}` ([#733](https://github.com/opensearch-project/opensearch-api-specification/pull/733)) +- Added `GET /_plugins/_ml/models/{model_id}`, `POST /_plugins/_ml/models/_search`, `POST /_plugins/_ml/models/_unload`, `_undeploy`, `_upload`, `meta`, `_register_meta`, `POST /_plugins/_ml/models/{model_id}/_load`, `_predict`, `_unload`, `chunk/{chunk_number}`, `upload_chunk/{chunk_number}`, and `PUT /_plugins/_ml/models/{model_id}` ([#733](https://github.com/opensearch-project/opensearch-api-specification/pull/733)) ### Removed - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652)) @@ -82,6 +82,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Fixed response schema for `GET /_plugins/_security/api/nodesdn/{cluster_name}` ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731)) - Fixed request body for `/_search/scroll/{scroll_id}` ([#732](https://github.com/opensearch-project/opensearch-api-specification/pull/732)) - Fixed type in `_msearch/template` ([#735](https://github.com/opensearch-project/opensearch-api-specification/pull/735)) +- Fixed indices API schemas ([#750](https://github.com/opensearch-project/opensearch-api-specification/pull/750)) ### Changed - Changed `tasks._common:TaskInfo` and `tasks._common:TaskGroup` to be composed of a `tasks._common:TaskInfoBase` ([#683](https://github.com/opensearch-project/opensearch-api-specification/pull/683)) diff --git a/spec/namespaces/ml.yaml b/spec/namespaces/ml.yaml index 8ab6a2a0c..b948bb948 100644 --- a/spec/namespaces/ml.yaml +++ b/spec/namespaces/ml.yaml @@ -65,19 +65,34 @@ paths: responses: '200': $ref: '#/components/responses/ml.register_model_meta@200' - /_plugins/_ml/models/{model_id}/chunk/{index}: + /_plugins/_ml/models/{model_id}/chunk/{chunk_number}: post: operationId: ml.chunk_model.0 x-operation-group: ml.chunk_model - description: Chunks model. + deprecated: true + x-deprecation-message: Use `upload_chunk` instead. + description: Uploads model chunk. parameters: - - $ref: '#/components/parameters/ml.chunk_model::path.index' + - $ref: '#/components/parameters/ml.chunk_model::path.chunk_number' - $ref: '#/components/parameters/ml.chunk_model::path.model_id' requestBody: $ref: '#/components/requestBodies/ml.chunk_model' responses: '200': $ref: '#/components/responses/ml.chunk_model@200' + /_plugins/_ml/models/{model_id}/upload_chunk/{chunk_number}: + post: + operationId: ml.upload_chunk.0 + x-operation-group: ml.upload_chunk + description: Uploads model chunk. + parameters: + - $ref: '#/components/parameters/ml.upload_chunk::path.chunk_number' + - $ref: '#/components/parameters/ml.upload_chunk::path.model_id' + requestBody: + $ref: '#/components/requestBodies/ml.upload_chunk' + responses: + '200': + $ref: '#/components/responses/ml.upload_chunk@200' /_plugins/_ml/models/_upload: post: operationId: ml.upload_model.0 @@ -418,6 +433,17 @@ components: description: The model chunk. required: - chunk + ml.upload_chunk: + content: + application/json: + schema: + type: object + properties: + chunk: + type: string + description: The model chunk. + required: + - chunk ml.upload_model: content: application/json: @@ -655,7 +681,18 @@ components: properties: status: type: string - description: The status of the chunking operation. + description: The status of the chunk upload operation. + required: + - status + ml.upload_chunk@200: + content: + application/json: + schema: + type: object + properties: + status: + type: string + description: The status of the chunk upload operation. required: - status ml.upload_model@200: @@ -843,8 +880,8 @@ components: required: true schema: type: string - ml.chunk_model::path.index: - name: index + ml.chunk_model::path.chunk_number: + name: chunk_number in: path required: true schema: @@ -856,6 +893,19 @@ components: required: true schema: type: string + ml.upload_chunk::path.chunk_number: + name: chunk_number + in: path + required: true + schema: + type: integer + format: int64 + ml.upload_chunk::path.model_id: + name: model_id + in: path + required: true + schema: + type: string ml.delete_task::path.task_id: name: task_id in: path diff --git a/tests/plugins/ml/models/create_metadata.yaml b/tests/plugins/ml/models/create_metadata.yaml index 4ad59ccd7..a0bb4ea6d 100644 --- a/tests/plugins/ml/models/create_metadata.yaml +++ b/tests/plugins/ml/models/create_metadata.yaml @@ -1,11 +1,12 @@ $schema: ../../../../json_schemas/test_story.schema.yaml -description: Test the creation of model metadata using meta API (deprecated). +description: Test deprecated meta and chunk APIs for model creation and chunk upload. distributions: excluded: - amazon-managed - amazon-serverless -version: '>= 2.11' +warnings: + multiple-paths-detected: false prologues: - path: /_cluster/settings method: PUT @@ -39,4 +40,43 @@ chapters: response: status: 200 output: - model_id: payload.model_id \ No newline at end of file + model_id: payload.model_id + - synopsis: Upload chunk 0. + path: /_plugins/_ml/models/{model_id}/chunk/{chunk_number} + method: POST + parameters: + chunk_number: 0 + model_id: ${model_metadata.model_id} + request: + payload: + chunk: chunk0 + response: + status: 200 + payload: + status: Uploaded + - synopsis: Upload chunk 1. + path: /_plugins/_ml/models/{model_id}/chunk/{chunk_number} + method: POST + parameters: + chunk_number: 1 + model_id: ${model_metadata.model_id} + request: + payload: + chunk: chunk1 + response: + status: 200 + payload: + status: Uploaded + - synopsis: Upload chunk 2. + path: /_plugins/_ml/models/{model_id}/chunk/{chunk_number} + method: POST + parameters: + chunk_number: 2 + model_id: ${model_metadata.model_id} + request: + payload: + chunk: chunk2 + response: + status: 200 + payload: + status: Uploaded diff --git a/tests/plugins/ml/models/register_metadata.yaml b/tests/plugins/ml/models/register_metadata.yaml index 30faf08a1..d300f6e8d 100644 --- a/tests/plugins/ml/models/register_metadata.yaml +++ b/tests/plugins/ml/models/register_metadata.yaml @@ -1,6 +1,6 @@ $schema: ../../../../json_schemas/test_story.schema.yaml -description: Test the creation of model metadata and chunking. +description: Test the creation of model metadata and upload model chunk. distributions: excluded: - amazon-managed @@ -43,10 +43,10 @@ chapters: output: model_id: payload.model_id - synopsis: Upload chunk 0. - path: /_plugins/_ml/models/{model_id}/chunk/{index} + path: /_plugins/_ml/models/{model_id}/upload_chunk/{chunk_number} method: POST parameters: - index: 0 + chunk_number: 0 model_id: ${model_metadata.model_id} request: payload: @@ -56,10 +56,10 @@ chapters: payload: status: Uploaded - synopsis: Upload chunk 1. - path: /_plugins/_ml/models/{model_id}/chunk/{index} + path: /_plugins/_ml/models/{model_id}/upload_chunk/{chunk_number} method: POST parameters: - index: 1 + chunk_number: 1 model_id: ${model_metadata.model_id} request: payload: @@ -69,10 +69,10 @@ chapters: payload: status: Uploaded - synopsis: Upload chunk 2. - path: /_plugins/_ml/models/{model_id}/chunk/{index} + path: /_plugins/_ml/models/{model_id}/upload_chunk/{chunk_number} method: POST parameters: - index: 2 + chunk_number: 2 model_id: ${model_metadata.model_id} request: payload: