Ensure correct OpenAPI 3.1.0 spec. (#646)

* Added Python and Ruby spec validators. Signed-off-by: dblock <[email protected]> * Add empty descriptions where missing and remove descriptions next to refs. Signed-off-by: dblock <[email protected]> * Replaced the validator compatible with OpenAPI 3.1. Signed-off-by: dblock <[email protected]> * Fixed incorrect style. Signed-off-by: dblock <[email protected]> * Added missing require. Signed-off-by: dblock <[email protected]> * Fix: schema null is not supported. Signed-off-by: dblock <[email protected]> * Removed unused required property. Signed-off-by: dblock <[email protected]> * Fix: added missing schema. Signed-off-by: dblock <[email protected]> * Fix: removed incorrect defaults. Signed-off-by: dblock <[email protected]> * Fix: added missing required. Signed-off-by: dblock <[email protected]> * Fix: removed incorrect required. Signed-off-by: dblock <[email protected]> * Removed incorrect externalDocs. Signed-off-by: dblock <[email protected]> * Document the ruby spec validator. Signed-off-by: dblock <[email protected]> * Fix: removed redundant content types. Signed-off-by: dblock <[email protected]> * Improve displaying of errors. Signed-off-by: dblock <[email protected]> * Normalize keys to be compatible with OpenAPI 3.1. Signed-off-by: dblock <[email protected]> * Removed superseeded operations that have unresolved parameters. Signed-off-by: dblock <[email protected]> * Fix: don't reduce error payload, allows to verify schema. Signed-off-by: dblock <[email protected]> * Fix: flow framework schema tests. Signed-off-by: dblock <[email protected]> * Fix: if no decision is made there's no weight ranking. Signed-off-by: dblock <[email protected]> * Fix: wrap indexing errors. Signed-off-by: dblock <[email protected]> * Added verbose logging of the story being evaluated. Signed-off-by: dblock <[email protected]> * Fix: security API test error responses. Signed-off-by: dblock <[email protected]> * Fix: duration ref. Signed-off-by: dblock <[email protected]> * Fix typo in property. Signed-off-by: dblock <[email protected]> * Fix superseeded operation parameters. Signed-off-by: dblock <[email protected]> * Fix: neural search error response. Signed-off-by: dblock <[email protected]> * Normalize fields. .replaceAll('::', '___') .replaceAll('@', '__') .replaceAll(':', '_') Signed-off-by: dblock <[email protected]> * Re-added default values into the schema. Signed-off-by: dblock <[email protected]> * Disallowed characters are replaced with ___. Signed-off-by: dblock <[email protected]> --------- Signed-off-by: dblock <[email protected]>
opensearch-project · Nov 6, 2024 · 2a39edd · 2a39edd
1 parent 3167577
commit 2a39edd
Show file tree

Hide file tree

Showing 47 changed files with 1,295 additions and 501 deletions.
diff --git a/.cspell b/.cspell
@@ -63,6 +63,7 @@ geotile
 gsub
 Gsub
 haasephonetik
+hashcode
 heteroscedastic
 hnsw
 homoscedastic
@@ -134,6 +135,7 @@ Oversample
 performanceanalyzer
 permissionsinfo
 pipefail
+pipenv
 preconfigure
 preconfigured
 prefilter

diff --git a/.github/workflows/validate-spec.yml → .github/workflows/validate-spec-lint.yml b/.github/workflows/validate-spec.yml → .github/workflows/validate-spec-lint.yml
diff --git a/.github/workflows/validate-spec-py.yml b/.github/workflows/validate-spec-py.yml
@@ -0,0 +1,34 @@
+name: Validate Spec (Python)
+
+on: [pull_request,push]
+
+jobs:
+  validate-spec-py:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+
+      - name: Build
+        run: npm ci && npm run merge
+
+      - name: Set Up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install Dependencies
+        working-directory: tools/src/validate-spec-py
+        run: |
+          pip install --user pipenv
+          pipenv install
+
+      - name: Validate Spec
+        working-directory: tools/src/validate-spec-py
+        run: |
+          pipenv run python validate.py ../../../build/opensearch-openapi.yaml
diff --git a/.github/workflows/validate-spec-ruby.yml b/.github/workflows/validate-spec-ruby.yml
@@ -0,0 +1,30 @@
+name: Validate Spec (Ruby)
+
+on: [pull_request, push]
+
+jobs:
+  validate-spec-ruby:
+    runs-on: ubuntu-latest
+    env:
+      BUNDLE_GEMFILE: ${{ github.workspace }}/tools/src/validate-spec-ruby/Gemfile
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+
+      - name: Build
+        run: npm ci && npm run merge
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+          bundler-cache: true
+
+      - name: Validate Spec
+        working-directory: tools/src/validate-spec-ruby
+        run: |
+          bundle exec ruby validate.rb ../../../build/opensearch-openapi.yaml
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,10 +6,22 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### Added
 - Added API specs for query groups lifecycle APIs ([#649](https://github.com/opensearch-project/opensearch-api-specification/pull/649))
+- Added Python and Ruby spec validators ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Added verbose output of the story being evaluated ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
 
 ### Removed
 - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))
 
+### Fixed
+- Spec passes OpenAPI 3.1.0 validations ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Disallowed characters (`::`, `@`, and `:`) in key names are replaced with `___` on merge ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Added missing `required` to `path` parameters ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Added missing `schema` parent to response types ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Removed invalid `externalDocs` from `flow_framework.create/update::query.use_case` ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Fixed incorrect `style` in `indices.get_mapping::query.index` ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Removed invalid `required` from `ppl` responses ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+- Added schema for security API error responses ([#646](https://github.com/opensearch-project/opensearch-api-specification/pull/646))
+
 ## [0.1.0] - 2024-10-25
 
 ### Added

diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -26,7 +26,9 @@
     - [Comment on PR](#comment-on-pr)
     - [Test Tools (Unit)](#test-tools-unit)
     - [Test Tools (Integration)](#test-tools-integration)
-    - [Validate Spec](#validate-spec)
+    - [Validate Spec (Lint)](#validate-spec-lint)
+    - [Validate Spec (Python)](#validate-spec-python)
+    - [Validate Spec (Ruby)](#validate-spec-ruby)
 <!-- TOC -->
 
 # Developer Guide
@@ -375,6 +377,30 @@ This workflow runs on PRs to invoke the [tools' unit tests](tools/tests), upload
 
 This workflow runs on PRs to invoke the [tools' integration tests](tools/tests) that require a running instance of OpenSearch to ensure there are no breakages in behavior.
 
-### [Validate Spec](.github/workflows/validate-spec.yml)
+### [Validate Spec (Lint)](.github/workflows/validate-spec-lint.yml)
 
 This workflow runs on PRs to invoke the [spec linter](#spec-linter) and ensure the multi-file spec is correct and follows the design guidelines.
+
+### [Validate Spec (Python)](.github/workflows/validate-spec-py.yml)
+
+This workflow runs on PRs to invoke the [Python openapi-spec-validator](https://pypi.org/project/openapi-spec-validator/) to ensure that the resulting spec can be loaded by Python tools.
+
+You can run the validator locally as follows after installing [pipenv](https://pipenv.pypa.io/en/latest/installation.html).
+
+```
+cd tools/src/validate-spec-py
+pipenv install
+npm run merge ; pipenv run python validate.py ../../../build/opensearch-openapi.yaml
+```
+
+### [Validate Spec (Ruby)](.github/workflows/validate-spec-ruby.yml)
+
+This workflow runs on PRs to invoke the Ruby [Json Schemer](https://github.com/davishmcclurg/json_schemer/) to ensure that the resulting spec can be loaded by Ruby tools.
+
+You can run the validator locally as follows.
+
+```
+cd tools/src/validate-spec-ruby
+bundle install
+npm run merge ; bundle exec ruby validate.rb ../../../build/opensearch-openapi.yaml
+```
diff --git a/spec/_global_parameters.yaml b/spec/_global_parameters.yaml
@@ -10,21 +10,21 @@ components:
       description: Whether to pretty format the returned JSON response.
       schema:
         type: boolean
-      default: false
+        default: false
     human:
       name: human
       in: query
       description: Whether to return human readable values for statistics.
       schema:
         type: boolean
-      default: true
+        default: true
     error_trace:
       name: error_trace
       in: query
       description: Whether to include the stack trace of returned errors.
       schema:
         type: boolean
-      default: false
+        default: false
     source:
       name: source
       in: query

diff --git a/spec/_superseded_operations.yaml b/spec/_superseded_operations.yaml
@@ -201,7 +201,7 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml
     - GET
     - PUT
 /_opendistro/_ism/policies/{policyID}:
-  superseded_by: /_plugins/_ism/policies/{policyID}
+  superseded_by: /_plugins/_ism/policies/{policy_id}
   operations:
     - GET
     - HEAD
@@ -240,7 +240,7 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml
   operations:
     - GET
 /_opendistro/_knn/{nodeId}/stats/{stat}:
-  superseded_by: /_plugins/_knn/{nodeId}/stats/{stat}
+  superseded_by: /_plugins/_knn/{node_id}/stats/{stat}
   operations:
     - GET
 /_opendistro/_performanceanalyzer/_agent/{redirectEndpoint}:
@@ -354,22 +354,22 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml
     - GET
     - PUT
 /_opendistro/_rollup/jobs/{rollupID}:
-  superseded_by: /_plugins/_rollup/jobs/{rollupID}
+  superseded_by: /_plugins/_rollup/jobs/{id}
   operations:
     - GET
     - HEAD
     - PUT
     - DELETE
 /_opendistro/_rollup/jobs/{rollupID}/_explain:
-  superseded_by: /_plugins/_rollup/jobs/{rollupID}/_explain
+  superseded_by: /_plugins/_rollup/jobs/{id}/_explain
   operations:
     - GET
 /_opendistro/_rollup/jobs/{rollupID}/_start:
-  superseded_by: /_plugins/_rollup/jobs/{rollupID}/_start
+  superseded_by: /_plugins/_rollup/jobs/{id}/_start
   operations:
     - POST
 /_opendistro/_rollup/jobs/{rollupID}/_stop:
-  superseded_by: /_plugins/_rollup/jobs/{rollupID}/_stop
+  superseded_by: /_plugins/_rollup/jobs/{id}/_stop
   operations:
     - POST
 /_opendistro/_security/api/_upgrade_check/:
@@ -401,7 +401,7 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml
     - GET
     - PATCH
 /_opendistro/_security/api/actiongroups/{name}:
-  superseded_by: /_plugins/_security/api/actiongroups/{name}
+  superseded_by: /_plugins/_security/api/actiongroups/{action_group}
   operations:
     - GET
     - PUT
@@ -433,14 +433,14 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml
     - GET
     - PATCH
 /_opendistro/_security/api/internalusers/{name}:
-  superseded_by: /_plugins/_security/api/internalusers/{name}
+  superseded_by: /_plugins/_security/api/internalusers/{username}
   operations:
     - GET
     - PUT
     - PATCH
     - DELETE
 /_opendistro/_security/api/internalusers/{name}/authtoken:
-  superseded_by: /_plugins/_security/api/internalusers/{name}/authtoken
+  superseded_by: /_plugins/_security/api/internalusers/{username}/authtoken
   operations:
     - POST
 /_opendistro/_security/api/migrate:
@@ -457,7 +457,7 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml
     - GET
     - PATCH
 /_opendistro/_security/api/roles/{name}:
-  superseded_by: /_plugins/_security/api/roles/{name}
+  superseded_by: /_plugins/_security/api/roles/{role}
   operations:
     - GET
     - PUT
@@ -469,7 +469,7 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml
     - GET
     - PATCH
 /_opendistro/_security/api/rolesmapping/{name}:
-  superseded_by: /_plugins/_security/api/rolesmapping/{name}
+  superseded_by: /_plugins/_security/api/rolesmapping/{role}
   operations:
     - GET
     - PUT
@@ -519,7 +519,7 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml
     - GET
     - PATCH
 /_opendistro/_security/api/tenants/{name}:
-  superseded_by: /_plugins/_security/api/tenants/{name}
+  superseded_by: /_plugins/_security/api/tenants/{tenant}
   operations:
     - GET
     - PUT
@@ -529,14 +529,14 @@ $schema: ./json_schemas/_superseded_operations.schema.yaml
   superseded_by: /_plugins/_security/api/user
   operations:
     - GET
-/_opendistro/_security/api/user/{name}:
-  superseded_by: /_plugins/_security/api/user/{name}
+/_opendistro/_security/api/user/{username}:
+  superseded_by: /_plugins/_security/api/user/{username}
   operations:
     - GET
     - PUT
     - DELETE
 /_opendistro/_security/api/user/{name}/authtoken:
-  superseded_by: /_plugins/_security/api/user/{name}/authtoken
+  superseded_by: /_plugins/_security/api/user/{username}/authtoken
   operations:
     - POST
 /_opendistro/_security/api/validate:

diff --git a/spec/namespaces/asynchronous_search.yaml b/spec/namespaces/asynchronous_search.yaml
@@ -90,11 +90,13 @@ components:
       in: path
       schema:
         type: string
+      required: true
     asynchronous_search.delete::path.id:
       name: id
       in: path
       schema:
         type: string
+      required: true
   requestBodies:
     asynchronous_search.search:
       content: