From 90c74aeef409c7021cf66a6583c1471f228ea6d5 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 11 Jan 2024 17:48:15 -0500 Subject: [PATCH] Ensure Jackson default maximums introduced in 2.16.0 do not conflict with OpenSearch settings (#11811) * Ensure Jackson default maximums introduced in 2.16.0 do not conflict with OpenSearch settings Signed-off-by: Andriy Redko * Address code review comments Signed-off-by: Andriy Redko * Address code review comments Signed-off-by: Andriy Redko --------- Signed-off-by: Andriy Redko (cherry picked from commit 5c82ab885a876d659c9714c3b080488777506027) Signed-off-by: Andriy Redko --- CHANGELOG.md | 1 + buildSrc/version.properties | 4 +- .../licenses/jackson-core-2.16.0.jar.sha1 | 1 - .../licenses/jackson-core-2.16.1.jar.sha1 | 1 + .../jackson-annotations-2.16.0.jar.sha1 | 1 - .../jackson-annotations-2.16.1.jar.sha1 | 1 + .../licenses/jackson-databind-2.16.0.jar.sha1 | 1 - .../licenses/jackson-databind-2.16.1.jar.sha1 | 1 + .../licenses/jackson-core-2.16.0.jar.sha1 | 1 - .../licenses/jackson-core-2.16.1.jar.sha1 | 1 + .../licenses/jackson-core-2.16.0.jar.sha1 | 1 - .../licenses/jackson-core-2.16.1.jar.sha1 | 1 + .../jackson-dataformat-cbor-2.16.0.jar.sha1 | 1 - .../jackson-dataformat-cbor-2.16.1.jar.sha1 | 1 + .../jackson-dataformat-smile-2.16.0.jar.sha1 | 1 - .../jackson-dataformat-smile-2.16.1.jar.sha1 | 1 + .../jackson-dataformat-yaml-2.16.0.jar.sha1 | 1 - .../jackson-dataformat-yaml-2.16.1.jar.sha1 | 1 + .../common/xcontent/XContentContraints.java | 37 +++++++ .../common/xcontent/cbor/CborXContent.java | 17 +-- .../common/xcontent/json/JsonXContent.java | 17 +-- .../common/xcontent/smile/SmileXContent.java | 17 +-- .../common/xcontent/yaml/YamlXContent.java | 17 +-- .../common/xcontent/XContentParserTests.java | 97 ++++++++++++++++++ .../common/xcontent/depth-off-limit.cbor.gz | Bin 0 -> 1888 bytes .../common/xcontent/depth-off-limit.json.gz | Bin 0 -> 2045 bytes .../common/xcontent/depth-off-limit.smile.gz | Bin 0 -> 1906 bytes .../common/xcontent/depth-off-limit.yaml.gz | Bin 0 -> 5950 bytes .../jackson-annotations-2.16.0.jar.sha1 | 1 - .../jackson-annotations-2.16.1.jar.sha1 | 1 + .../licenses/jackson-databind-2.16.0.jar.sha1 | 1 - .../licenses/jackson-databind-2.16.1.jar.sha1 | 1 + .../jackson-annotations-2.16.0.jar.sha1 | 1 - .../jackson-annotations-2.16.1.jar.sha1 | 1 + .../licenses/jackson-databind-2.16.0.jar.sha1 | 1 - .../licenses/jackson-databind-2.16.1.jar.sha1 | 1 + .../jackson-annotations-2.16.0.jar.sha1 | 1 - .../jackson-annotations-2.16.1.jar.sha1 | 1 + .../licenses/jackson-databind-2.16.0.jar.sha1 | 1 - .../licenses/jackson-databind-2.16.1.jar.sha1 | 1 + .../jackson-annotations-2.16.0.jar.sha1 | 1 - .../jackson-annotations-2.16.1.jar.sha1 | 1 + .../licenses/jackson-databind-2.16.0.jar.sha1 | 1 - .../licenses/jackson-databind-2.16.1.jar.sha1 | 1 + .../jackson-dataformat-xml-2.16.0.jar.sha1 | 1 - .../jackson-dataformat-xml-2.16.1.jar.sha1 | 1 + .../jackson-datatype-jsr310-2.16.0.jar.sha1 | 1 - .../jackson-datatype-jsr310-2.16.1.jar.sha1 | 1 + ...on-module-jaxb-annotations-2.16.0.jar.sha1 | 1 - ...on-module-jaxb-annotations-2.16.1.jar.sha1 | 1 + .../jackson-annotations-2.16.0.jar.sha1 | 1 - .../jackson-annotations-2.16.1.jar.sha1 | 1 + .../licenses/jackson-databind-2.16.0.jar.sha1 | 1 - .../licenses/jackson-databind-2.16.1.jar.sha1 | 1 + .../index/mapper/MapperService.java | 35 ++++++- .../index/mapper/MapperServiceTests.java | 17 +++ 56 files changed, 253 insertions(+), 48 deletions(-) delete mode 100644 client/sniffer/licenses/jackson-core-2.16.0.jar.sha1 create mode 100644 client/sniffer/licenses/jackson-core-2.16.1.jar.sha1 delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-annotations-2.16.0.jar.sha1 create mode 100644 distribution/tools/upgrade-cli/licenses/jackson-annotations-2.16.1.jar.sha1 delete mode 100644 distribution/tools/upgrade-cli/licenses/jackson-databind-2.16.0.jar.sha1 create mode 100644 distribution/tools/upgrade-cli/licenses/jackson-databind-2.16.1.jar.sha1 delete mode 100644 libs/core/licenses/jackson-core-2.16.0.jar.sha1 create mode 100644 libs/core/licenses/jackson-core-2.16.1.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-core-2.16.0.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-core-2.16.1.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.16.0.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-cbor-2.16.1.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.16.0.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-smile-2.16.1.jar.sha1 delete mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.16.0.jar.sha1 create mode 100644 libs/x-content/licenses/jackson-dataformat-yaml-2.16.1.jar.sha1 create mode 100644 libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentContraints.java create mode 100644 libs/x-content/src/test/resources/org/opensearch/common/xcontent/depth-off-limit.cbor.gz create mode 100644 libs/x-content/src/test/resources/org/opensearch/common/xcontent/depth-off-limit.json.gz create mode 100644 libs/x-content/src/test/resources/org/opensearch/common/xcontent/depth-off-limit.smile.gz create mode 100644 libs/x-content/src/test/resources/org/opensearch/common/xcontent/depth-off-limit.yaml.gz delete mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.16.0.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/jackson-annotations-2.16.1.jar.sha1 delete mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.16.0.jar.sha1 create mode 100644 modules/ingest-geoip/licenses/jackson-databind-2.16.1.jar.sha1 delete mode 100644 plugins/crypto-kms/licenses/jackson-annotations-2.16.0.jar.sha1 create mode 100644 plugins/crypto-kms/licenses/jackson-annotations-2.16.1.jar.sha1 delete mode 100644 plugins/crypto-kms/licenses/jackson-databind-2.16.0.jar.sha1 create mode 100644 plugins/crypto-kms/licenses/jackson-databind-2.16.1.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.16.0.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/jackson-annotations-2.16.1.jar.sha1 delete mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.16.0.jar.sha1 create mode 100644 plugins/discovery-ec2/licenses/jackson-databind-2.16.1.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-annotations-2.16.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-annotations-2.16.1.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-databind-2.16.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-databind-2.16.1.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-dataformat-xml-2.16.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-dataformat-xml-2.16.1.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-datatype-jsr310-2.16.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-datatype-jsr310-2.16.1.jar.sha1 delete mode 100644 plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.16.0.jar.sha1 create mode 100644 plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.16.1.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.16.0.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-annotations-2.16.1.jar.sha1 delete mode 100644 plugins/repository-s3/licenses/jackson-databind-2.16.0.jar.sha1 create mode 100644 plugins/repository-s3/licenses/jackson-databind-2.16.1.jar.sha1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 34ced9000a6de..6b72a016494eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -105,6 +105,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add deleted doc count in _cat/shards ([#11678](https://github.com/opensearch-project/OpenSearch/pull/11678)) - Capture information for additional query types and aggregation types ([#11582](https://github.com/opensearch-project/OpenSearch/pull/11582)) - Use slice_size == shard_size heuristic in terms aggs for concurrent segment search and properly calculate the doc_count_error ([#11732](https://github.com/opensearch-project/OpenSearch/pull/11732)) +- Ensure Jackson default maximums introduced in 2.16.0 do not conflict with OpenSearch settings ([#11890](https://github.com/opensearch-project/OpenSearch/pull/11890)) ### Deprecated diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 64550370e22dd..3579fb8e16b92 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -7,8 +7,8 @@ bundled_jdk = 21.0.1+12 # optional dependencies spatial4j = 0.7 jts = 1.15.0 -jackson = 2.16.0 -jackson_databind = 2.16.0 +jackson = 2.16.1 +jackson_databind = 2.16.1 snakeyaml = 2.1 icu4j = 70.1 supercsv = 2.4.0 diff --git a/client/sniffer/licenses/jackson-core-2.16.0.jar.sha1 b/client/sniffer/licenses/jackson-core-2.16.0.jar.sha1 deleted file mode 100644 index c2b70fb4ae202..0000000000000 --- a/client/sniffer/licenses/jackson-core-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -899e5cf01be55fbf094ad72b2edb0c5df99111ee \ No newline at end of file diff --git a/client/sniffer/licenses/jackson-core-2.16.1.jar.sha1 b/client/sniffer/licenses/jackson-core-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..908d071b34a2a --- /dev/null +++ b/client/sniffer/licenses/jackson-core-2.16.1.jar.sha1 @@ -0,0 +1 @@ +9456bb3cdd0f79f91a5f730a1b1bb041a380c91f \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.16.0.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.16.0.jar.sha1 deleted file mode 100644 index 79ed9e0c63fc8..0000000000000 --- a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc30995f7428c0a405eba9b8c619b20d2b3b9905 \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.16.1.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..cbc65687606fc --- /dev/null +++ b/distribution/tools/upgrade-cli/licenses/jackson-annotations-2.16.1.jar.sha1 @@ -0,0 +1 @@ +fd441d574a71e7d10a4f73de6609f881d8cdfeec \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.16.0.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.16.0.jar.sha1 deleted file mode 100644 index da00d281934b1..0000000000000 --- a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3a6b7f8ff7b30d518bbd65678e9c30cd881f19a7 \ No newline at end of file diff --git a/distribution/tools/upgrade-cli/licenses/jackson-databind-2.16.1.jar.sha1 b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..d231db4fd49fc --- /dev/null +++ b/distribution/tools/upgrade-cli/licenses/jackson-databind-2.16.1.jar.sha1 @@ -0,0 +1 @@ +02a16efeb840c45af1e2f31753dfe76795278b73 \ No newline at end of file diff --git a/libs/core/licenses/jackson-core-2.16.0.jar.sha1 b/libs/core/licenses/jackson-core-2.16.0.jar.sha1 deleted file mode 100644 index c2b70fb4ae202..0000000000000 --- a/libs/core/licenses/jackson-core-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -899e5cf01be55fbf094ad72b2edb0c5df99111ee \ No newline at end of file diff --git a/libs/core/licenses/jackson-core-2.16.1.jar.sha1 b/libs/core/licenses/jackson-core-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..908d071b34a2a --- /dev/null +++ b/libs/core/licenses/jackson-core-2.16.1.jar.sha1 @@ -0,0 +1 @@ +9456bb3cdd0f79f91a5f730a1b1bb041a380c91f \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-core-2.16.0.jar.sha1 b/libs/x-content/licenses/jackson-core-2.16.0.jar.sha1 deleted file mode 100644 index c2b70fb4ae202..0000000000000 --- a/libs/x-content/licenses/jackson-core-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -899e5cf01be55fbf094ad72b2edb0c5df99111ee \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-core-2.16.1.jar.sha1 b/libs/x-content/licenses/jackson-core-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..908d071b34a2a --- /dev/null +++ b/libs/x-content/licenses/jackson-core-2.16.1.jar.sha1 @@ -0,0 +1 @@ +9456bb3cdd0f79f91a5f730a1b1bb041a380c91f \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.16.0.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.16.0.jar.sha1 deleted file mode 100644 index 8da478fc6013d..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-cbor-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -35e8b7bf4fc1d078766bb155103d433ed5bb1627 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-cbor-2.16.1.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-cbor-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..b4b781f604910 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-cbor-2.16.1.jar.sha1 @@ -0,0 +1 @@ +1be7098dccc079171464dca7e386bd8df623b031 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.16.0.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.16.0.jar.sha1 deleted file mode 100644 index 3e952ffe92418..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-smile-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3c422d7f3901c9a1becf9df3cf41efc68a5ab95c \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-smile-2.16.1.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-smile-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..ad91e748ebe94 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-smile-2.16.1.jar.sha1 @@ -0,0 +1 @@ +c4ddbc5277670f2e56b1f5e44e83afa748bcb125 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.16.0.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.16.0.jar.sha1 deleted file mode 100644 index d62b5874ab023..0000000000000 --- a/libs/x-content/licenses/jackson-dataformat-yaml-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2033e2c5f531785d17f3a2bc31842e3bbb7983b2 \ No newline at end of file diff --git a/libs/x-content/licenses/jackson-dataformat-yaml-2.16.1.jar.sha1 b/libs/x-content/licenses/jackson-dataformat-yaml-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..9b30e7bf921b2 --- /dev/null +++ b/libs/x-content/licenses/jackson-dataformat-yaml-2.16.1.jar.sha1 @@ -0,0 +1 @@ +8e4f1923d73cd55f2b4c0d56ee4ed80419297354 \ No newline at end of file diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentContraints.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentContraints.java new file mode 100644 index 0000000000000..0b80bec8577d0 --- /dev/null +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/XContentContraints.java @@ -0,0 +1,37 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.common.xcontent; + +import com.fasterxml.jackson.core.StreamReadConstraints; + +import org.opensearch.common.annotation.InternalApi; + +/** + * Consolidates the XContent constraints (primarily reflecting Jackson's {@link StreamReadConstraints} constraints) + * + * @opensearch.internal + */ +@InternalApi +public interface XContentContraints { + final String DEFAULT_MAX_STRING_LEN_PROPERTY = "opensearch.xcontent.string.length.max"; + final String DEFAULT_MAX_NAME_LEN_PROPERTY = "opensearch.xcontent.name.length.max"; + final String DEFAULT_MAX_DEPTH_PROPERTY = "opensearch.xcontent.depth.max"; + + final int DEFAULT_MAX_STRING_LEN = Integer.parseInt( + System.getProperty(DEFAULT_MAX_STRING_LEN_PROPERTY, Integer.toString(Integer.MAX_VALUE) /* no limit */ ) + ); + + final int DEFAULT_MAX_NAME_LEN = Integer.parseInt( + System.getProperty(DEFAULT_MAX_NAME_LEN_PROPERTY, Integer.toString(Integer.MAX_VALUE) /* no limit */ ) + ); + + final int DEFAULT_MAX_DEPTH = Integer.parseInt( + System.getProperty(DEFAULT_MAX_DEPTH_PROPERTY, Integer.toString(Integer.MAX_VALUE) /* no limit */ ) + ); +} diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/cbor/CborXContent.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/cbor/CborXContent.java index cf6123dba57cd..d1e3279259ffa 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/cbor/CborXContent.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/cbor/CborXContent.java @@ -37,9 +37,11 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.core.StreamWriteConstraints; import com.fasterxml.jackson.core.StreamWriteFeature; import com.fasterxml.jackson.dataformat.cbor.CBORFactory; +import org.opensearch.common.xcontent.XContentContraints; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; @@ -59,11 +61,7 @@ /** * A CBOR based content implementation using Jackson. */ -public class CborXContent implements XContent { - public static final int DEFAULT_MAX_STRING_LEN = Integer.parseInt( - System.getProperty("opensearch.xcontent.string.length.max", Integer.toString(Integer.MAX_VALUE) /* no limit */) - ); - +public class CborXContent implements XContent, XContentContraints { public static final boolean USE_FAST_DOUBLE_WRITER = Boolean.getBoolean("opensearch.xcontent.use_fast_double_writer"); public static XContentBuilder contentBuilder() throws IOException { @@ -79,7 +77,14 @@ public static XContentBuilder contentBuilder() throws IOException { // Do not automatically close unclosed objects/arrays in com.fasterxml.jackson.dataformat.cbor.CBORGenerator#close() method cborFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false); cborFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true); - cborFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build()); + cborFactory.setStreamWriteConstraints(StreamWriteConstraints.builder().maxNestingDepth(DEFAULT_MAX_DEPTH).build()); + cborFactory.setStreamReadConstraints( + StreamReadConstraints.builder() + .maxStringLength(DEFAULT_MAX_STRING_LEN) + .maxNameLength(DEFAULT_MAX_NAME_LEN) + .maxNestingDepth(DEFAULT_MAX_DEPTH) + .build() + ); cborFactory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true); cborFactory.configure(StreamWriteFeature.USE_FAST_DOUBLE_WRITER.mappedFeature(), USE_FAST_DOUBLE_WRITER); cborXContent = new CborXContent(); diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContent.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContent.java index 3cd8af551ba81..03e17b7fc768c 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContent.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/json/JsonXContent.java @@ -38,8 +38,10 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.core.StreamWriteConstraints; import com.fasterxml.jackson.core.StreamWriteFeature; +import org.opensearch.common.xcontent.XContentContraints; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; @@ -58,11 +60,7 @@ /** * A JSON based content implementation using Jackson. */ -public class JsonXContent implements XContent { - public static final int DEFAULT_MAX_STRING_LEN = Integer.parseInt( - System.getProperty("opensearch.xcontent.string.length.max", Integer.toString(Integer.MAX_VALUE) /* no limit */) - ); - +public class JsonXContent implements XContent, XContentContraints { public static final boolean USE_FAST_DOUBLE_WRITER = Boolean.getBoolean("opensearch.xcontent.use_fast_double_writer"); public static XContentBuilder contentBuilder() throws IOException { @@ -81,7 +79,14 @@ public static XContentBuilder contentBuilder() throws IOException { // Do not automatically close unclosed objects/arrays in com.fasterxml.jackson.core.json.UTF8JsonGenerator#close() method jsonFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false); jsonFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true); - jsonFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build()); + jsonFactory.setStreamWriteConstraints(StreamWriteConstraints.builder().maxNestingDepth(DEFAULT_MAX_DEPTH).build()); + jsonFactory.setStreamReadConstraints( + StreamReadConstraints.builder() + .maxStringLength(DEFAULT_MAX_STRING_LEN) + .maxNameLength(DEFAULT_MAX_NAME_LEN) + .maxNestingDepth(DEFAULT_MAX_DEPTH) + .build() + ); jsonFactory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true); jsonFactory.configure(StreamWriteFeature.USE_FAST_DOUBLE_WRITER.mappedFeature(), USE_FAST_DOUBLE_WRITER); jsonXContent = new JsonXContent(); diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/smile/SmileXContent.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/smile/SmileXContent.java index b28192d0ab44f..0d6a90f8f1c43 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/smile/SmileXContent.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/smile/SmileXContent.java @@ -37,10 +37,12 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.core.StreamWriteConstraints; import com.fasterxml.jackson.core.StreamWriteFeature; import com.fasterxml.jackson.dataformat.smile.SmileFactory; import com.fasterxml.jackson.dataformat.smile.SmileGenerator; +import org.opensearch.common.xcontent.XContentContraints; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; @@ -59,11 +61,7 @@ /** * A Smile based content implementation using Jackson. */ -public class SmileXContent implements XContent { - public static final int DEFAULT_MAX_STRING_LEN = Integer.parseInt( - System.getProperty("opensearch.xcontent.string.length.max", Integer.toString(Integer.MAX_VALUE) /* no limit */) - ); - +public class SmileXContent implements XContent, XContentContraints { public static final boolean USE_FAST_DOUBLE_WRITER = Boolean.getBoolean("opensearch.xcontent.use_fast_double_writer"); public static XContentBuilder contentBuilder() throws IOException { @@ -81,7 +79,14 @@ public static XContentBuilder contentBuilder() throws IOException { // Do not automatically close unclosed objects/arrays in com.fasterxml.jackson.dataformat.smile.SmileGenerator#close() method smileFactory.configure(JsonGenerator.Feature.AUTO_CLOSE_JSON_CONTENT, false); smileFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true); - smileFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build()); + smileFactory.setStreamWriteConstraints(StreamWriteConstraints.builder().maxNestingDepth(DEFAULT_MAX_DEPTH).build()); + smileFactory.setStreamReadConstraints( + StreamReadConstraints.builder() + .maxStringLength(DEFAULT_MAX_STRING_LEN) + .maxNameLength(DEFAULT_MAX_NAME_LEN) + .maxNestingDepth(DEFAULT_MAX_DEPTH) + .build() + ); smileFactory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true); smileFactory.configure(StreamWriteFeature.USE_FAST_DOUBLE_WRITER.mappedFeature(), USE_FAST_DOUBLE_WRITER); smileXContent = new SmileXContent(); diff --git a/libs/x-content/src/main/java/org/opensearch/common/xcontent/yaml/YamlXContent.java b/libs/x-content/src/main/java/org/opensearch/common/xcontent/yaml/YamlXContent.java index 84e19b226cd36..661f22d389036 100644 --- a/libs/x-content/src/main/java/org/opensearch/common/xcontent/yaml/YamlXContent.java +++ b/libs/x-content/src/main/java/org/opensearch/common/xcontent/yaml/YamlXContent.java @@ -36,9 +36,11 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.StreamReadConstraints; import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.core.StreamWriteConstraints; import com.fasterxml.jackson.core.StreamWriteFeature; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.opensearch.common.xcontent.XContentContraints; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.MediaType; @@ -57,11 +59,7 @@ /** * A YAML based content implementation using Jackson. */ -public class YamlXContent implements XContent { - public static final int DEFAULT_MAX_STRING_LEN = Integer.parseInt( - System.getProperty("opensearch.xcontent.string.length.max", Integer.toString(Integer.MAX_VALUE) /* no limit */) - ); - +public class YamlXContent implements XContent, XContentContraints { public static final boolean USE_FAST_DOUBLE_WRITER = Boolean.getBoolean("opensearch.xcontent.use_fast_double_writer"); public static XContentBuilder contentBuilder() throws IOException { @@ -74,7 +72,14 @@ public static XContentBuilder contentBuilder() throws IOException { static { yamlFactory = new YAMLFactory(); yamlFactory.configure(JsonParser.Feature.STRICT_DUPLICATE_DETECTION, true); - yamlFactory.setStreamReadConstraints(StreamReadConstraints.builder().maxStringLength(DEFAULT_MAX_STRING_LEN).build()); + yamlFactory.setStreamWriteConstraints(StreamWriteConstraints.builder().maxNestingDepth(DEFAULT_MAX_DEPTH).build()); + yamlFactory.setStreamReadConstraints( + StreamReadConstraints.builder() + .maxStringLength(DEFAULT_MAX_STRING_LEN) + .maxNameLength(DEFAULT_MAX_NAME_LEN) + .maxNestingDepth(DEFAULT_MAX_DEPTH) + .build() + ); yamlFactory.configure(StreamReadFeature.USE_FAST_DOUBLE_PARSER.mappedFeature(), true); yamlFactory.configure(StreamWriteFeature.USE_FAST_DOUBLE_WRITER.mappedFeature(), USE_FAST_DOUBLE_WRITER); yamlXContent = new YamlXContent(); diff --git a/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java index eae7e522431de..5ca8b94c9f807 100644 --- a/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java +++ b/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java @@ -39,6 +39,7 @@ import org.opensearch.common.xcontent.cbor.CborXContent; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.common.xcontent.smile.SmileXContent; +import org.opensearch.common.xcontent.yaml.YamlXContent; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParseException; @@ -81,6 +82,28 @@ public class XContentParserTests extends OpenSearchTestCase { () -> randomAlphaOfLengthBetween(1, 3140000) ); + private static final Map> FIELD_NAME_GENERATORS = Map.of( + XContentType.JSON, + () -> randomAlphaOfLengthBetween(1, Math.min(JsonXContent.DEFAULT_MAX_NAME_LEN, 100_000)), /* limit to 100000, OOM otherwise */ + XContentType.CBOR, + () -> randomAlphaOfLengthBetween(1, Math.min(CborXContent.DEFAULT_MAX_NAME_LEN, 100_000)), /* limit to 100000, OOM otherwise */ + XContentType.SMILE, + () -> randomAlphaOfLengthBetween(1, Math.min(SmileXContent.DEFAULT_MAX_NAME_LEN, 100_000)), /* limit to 100000, OOM otherwise */ + XContentType.YAML, + () -> randomAlphaOfLengthBetween(1, Math.min(YamlXContent.DEFAULT_MAX_NAME_LEN, 100_000)) /* limit to 100000, OOM otherwise */ + ); + + private static final Map> DEPTH_GENERATORS = Map.of( + XContentType.JSON, + () -> randomIntBetween(1, Math.min(JsonXContent.DEFAULT_MAX_DEPTH, 1000)), /* limit to 1000, OOM otherwise */ + XContentType.CBOR, + () -> randomIntBetween(1, Math.min(CborXContent.DEFAULT_MAX_DEPTH, 1000)), /* limit to 1000, OOM otherwise */ + XContentType.SMILE, + () -> randomIntBetween(1, Math.min(SmileXContent.DEFAULT_MAX_DEPTH, 1000)), /* limit to 1000, OOM otherwise */ + XContentType.YAML, + () -> randomIntBetween(1, Math.min(YamlXContent.DEFAULT_MAX_DEPTH, 1000)) /* limit to 1000, OOM otherwise */ + ); + public void testStringOffLimit() throws IOException { final String field = randomAlphaOfLengthBetween(1, 5); final String value = randomRealisticUnicodeOfCodepointLength(3145730); @@ -136,6 +159,80 @@ public void testString() throws IOException { } } + public void testFieldName() throws IOException { + final XContentType xContentType = randomFrom(XContentType.values()); + + final String field = FIELD_NAME_GENERATORS.get(xContentType).get(); + final String value = randomAlphaOfLengthBetween(1, 5); + + try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) { + builder.startObject(); + if (randomBoolean()) { + builder.field(field, value); + } else { + builder.field(field).value(value); + } + builder.endObject(); + + try (XContentParser parser = createParser(xContentType.xContent(), BytesReference.bytes(builder))) { + assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); + assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); + assertEquals(field, parser.currentName()); + assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken()); + assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); + assertNull(parser.nextToken()); + } + } + } + + public void testDepth() throws IOException { + final XContentType xContentType = randomFrom(XContentType.values()); + + final String field = randomAlphaOfLengthBetween(1, 5); + final String value = randomAlphaOfLengthBetween(1, 5); + + try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) { + final int maxDepth = DEPTH_GENERATORS.get(xContentType).get() - 1; + + for (int depth = 0; depth < maxDepth; ++depth) { + builder.startObject(); + builder.field(field + depth); + } + + builder.startObject(); + if (randomBoolean()) { + builder.field(field, value); + } else { + builder.field(field).value(value); + } + builder.endObject(); + + for (int depth = 0; depth < maxDepth; ++depth) { + builder.endObject(); + } + + try (XContentParser parser = createParser(xContentType.xContent(), BytesReference.bytes(builder))) { + for (int depth = 0; depth < maxDepth; ++depth) { + assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); + assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); + assertEquals(field + depth, parser.currentName()); + } + + assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken()); + assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken()); + assertEquals(field, parser.currentName()); + assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken()); + assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); + + for (int depth = 0; depth < maxDepth; ++depth) { + assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken()); + } + + assertNull(parser.nextToken()); + } + } + } + public void testFloat() throws IOException { final XContentType xContentType = randomFrom(XContentType.values()); diff --git a/libs/x-content/src/test/resources/org/opensearch/common/xcontent/depth-off-limit.cbor.gz b/libs/x-content/src/test/resources/org/opensearch/common/xcontent/depth-off-limit.cbor.gz new file mode 100644 index 0000000000000000000000000000000000000000..88de7e590e7f0e785fb1462c78ddac6aed50e9df GIT binary patch literal 1888 zcmb2|=HS>^IX9JwIVH8ABtthpEloEkGdHtDFF7f{h~e$!-GV8{1zjDdm%X}vRmtQ6 z4~vJENs+H|x<%^?m1OC z^|RLV&~vLn0-LbFqWGu#PSsBRt@S$t{vlyER46|A_Wr}iA3pw2_`{+`#%?HalYd6oP3wO;RH(Zp{|Qt~p>%Zi$H>fn*F&Q*vSIU|~IF19)2(_I_= zv|3DxjMaURe*R0*k>p2`UcSH8I(<*NHU9YAoRmLi*?2tuv1$8k;eV!k*12eDh}m*T z8&XkFGxqs_;IHITRS<4-My{lv4NY@;wR-aleV6KjD2mKz*BrA&dh=s_`20iFx1yE| z2BhIYW?S@p;ruz@ukUc5LA~SDlRKPWgZdHGQ&9_Ggjd-kqKVEU_aAr<;S3R_;a%C^I67#S`{f?jw1EUTDTfiFMp4u#!@EX%p zv~JZVbkK@`Q$P|}=lkfRoZ3y7q%q|=LD2z1AO&nJ`kS&pA+j+@O-`0m0)7Mbg4caj zwFltH*Fis&KGU-0B?0#VnqRX_?UQ?Yb4u{F(q8fvd24_MV1lhh{gk|fsK($YrNgEi z+Bm=gAzy7R1CDwf?6Yjiy%A!9)KB7%u14(yR)b}-2QZb1kH%Ejk#TZFWP$o{H%u3j z+q9&)@!=Mtw0&l-4;QfjwawHwDx9DjDk;>45o{)RU>7%&2a#Iad-t;je;T6Ph77&* zJBjz$XTj1ubFj{e8g2l29cpr0PD>@yu0eBxgFnMN26*4N8MOtn*DBesF zFMeV{tB3BJwtd~kKg&C3xJ$hlYKeZ}oKR+^(rhM#m^d{?^wQq3P_KJv9mp15Hj(~j zABW`|?OWv-$AWLPEutWvim9eVpiuxQb6-j1%yJAOL6@kbKBali*oaK?U~-j<(#4R- z!!v0r)$b-~XB*=v?~YzfA{$-e62rPWyhnvSzh&8cRpBmmOjft+!U!(Nxk4W+dvW!^ zlv!dL+Mq*)f)6P8O4s#5*CfRwl2Gt7?$ug_$Bdo0f!(u);HDf7W%CLeP$r{BU=+fe zlPBx5*2hs_uI*$gF7<=wgtsRff40BvSQugLVwO8UU6-Qw?#E8vF=KmZ<`^$8uGV04 z%sV<$ie+6eok?iL^rP~CDDZ+Tj+1^xfsDFfVcajf5mRMqz=khMIP`&_-MM_4fRa3aQ>YHZ6|zSUiBM=oL;!@RUjS$Dj(PcvbmU$8%c zjgCcOf#u){*S{nTsM(! zQ@aCulR@*|8hbg!)0q=ZAsk<2@X3sAlgplqe@Zb%`n&gWXw8L*UfO)__m9MZoq<5H zY~M}vde?Pea&J-k>dqb}vSc zGE`n@Z*dF_XkryXC_^qq($}*mI+8`nmdNmaTDiBO6hnHIey+c)N|ZAJ3Bo+F)=?s OLC#3j&t4Dp@%bMjq~}5a literal 0 HcmV?d00001 diff --git a/libs/x-content/src/test/resources/org/opensearch/common/xcontent/depth-off-limit.smile.gz b/libs/x-content/src/test/resources/org/opensearch/common/xcontent/depth-off-limit.smile.gz new file mode 100644 index 0000000000000000000000000000000000000000..e248778b372539560625af3aba6b97a801a4094b GIT binary patch literal 1906 zcmb2|=HS>^IX9JwIVH8ABtthpEloEkGdHtDuQ)d|CzavtC2gTxPr+#|6h*V|M_!WfB(N9^Ui-i^8Ly8E8o9- zKl6Rg`@twZ-3tQJoo(VdD}q(W%n`Rw|#GGZ~xx*J@@_Xd)xQsSKkK;?6v>@dF#i+ zpFg&!+dj=aU(uItt0h0R|CIaF#h(IyuBZvI1981pw4k6>OT3WL$|+vRX!RB^By?&; z4+;vj)N+5iLT2jXqL98*K+ft-C@B2ts+g&(i$d3(3f-v{-g-B|{047X^N^uZ%y0UZ zO};x+sHE*Tbj!N$4izeG`_0_4*>{Hul~{g5wygc`P@xXLOZ>i}`iAWrxibFU?RSR~ zH@0m0-PH1%LxtM=Caz3>_w>7`-=%)vRDE*@alf1I{{QFc_5FYT=Kufky1xE@9T5Ke QzyCjDR=sb+133l;0Q8?wRsaA1 literal 0 HcmV?d00001 diff --git a/libs/x-content/src/test/resources/org/opensearch/common/xcontent/depth-off-limit.yaml.gz b/libs/x-content/src/test/resources/org/opensearch/common/xcontent/depth-off-limit.yaml.gz new file mode 100644 index 0000000000000000000000000000000000000000..3b36594482a6818c5d2f230aa862c21297f0cac8 GIT binary patch literal 5950 zcmWOAc{~$*90%~MEuy?(FXU*GTR_kMj}zpq9b3iTqUz#j_p^N%0~9uB{J`EY1Z zSP)Ss#wRQk%>S@`xQDq7a99qHr&uGC*uXiK?3jFX$Z-6)PVI2Ty$|?H5@VZps~2?i z#L!Q+?@F-TSe|6YZ#J$hGDqV#m%N8^He*&S-wbVK%i8AVXwK%*KkFL0n=!HLMY{13 zq%|?!cw*e9jc$DO%5sWseC+z~3~oZ0#b%x$YN%#IXMKKQuEjpSrDbVp##?n^Gffbl zKDwSKsBswGtQ7D%Gyc30=;~&ye-VV&WNZu>K#ejs=L9^a-3Dbfb-1qRlx4m5s{Tx= z%$N9s>p!k`bd?lgtJm5E^;3QSX@1m6z1&mP_#d`J5e0|Y93;(4*rNv#} zrvr)=5be%N>5b2UA)ceL%Wr5+tKUQ|C-Mz{;)3yKRw!5 zE%cljx*=>nOc`zdXQ?C{+q?czQ1EGR^ii|Wb<=~pROk^KzWR?{Ubw?3B}_OxBTnWn z3~`sjJ>ud`6FrCaocdH7PLZ0~FEgL9TkX?}l*v;z*JdYA^}NU@n>-{MFE6f}jCf_TB>&sUQoWd!q4;b1N$idD4mw|^6hHDxy}guM>PKE5jmuy~#mK}l z*Z+a8P?m=5enyRn>12(jX*A*E+~WAwE9)!ccE6%#5sF`U-_vw*Mp=qpn3fD(mY+?_ z)0$U-apB&P-~Z^e{P%8UJx*A%XtpV@*ma%llG9b+nytuLe8ZypLL|22Z=uB8V{Np0=n=B z-RDzfPjUw8&yGv}j;UPUCz3ihzMHI5dF_#T<=G}q;b?_ta2Iih?#rpSxjAbS-)sIu zEMeW(v&bhyBf8Cvjb8sQ+^6Ch-Idd!!wTr6DUMzbvUs`#&?|E?aq;DRF7SLr7= zdG@O(4rAuFjbC|Wc$;e6(RAYT(B;E>rtj7_Zxi*OUqr<&RwW&8rkI87X$@xxv-nJo z^54L?yXkK^X7`0JXycyMFBoP|?pK^K*R;cl$sOz<826Go1DV9|y8-OH;NgJJL&1jw zIVvHPz;7NQ9f4zcA!Lm^>7e3ejE(6UFUbZ}x`sAn*3D7N;TVf)>eafTh$FP|B9 zcf9O0?3;fnGW;&hR5uzv%sgo{ZqK}8^ec?{&}il^v)O2_+VaBnyybs9Qu0IVH`=Pg2Y`mdcgjHu$(U7L1a z^cRl3kohX#_Gu8~D2<-`fHg_3(p(gv>z?CjK7Q4Y4#C=2^KdNSG} z1Ab~w{q^nG;;O6J6sPk`&)BtfJ00#N_*3$&kA2BC`GCxcB;J;;S^qd_dP0mYydfcb zB!sV^;n)AG9P_oR&&8i{zNNIsss5Q`W9#LDqT6j9gO+E-@|PgJYS%n?s4|P`x5kXt z+C`lNEm+qYLRs3Jry6Lq{5FL#&R@Ke_zcEblTenm;$Z?UP^xRVr*<-oXWyH-_m$-L0%8xf?8 zW1rf%Z`7L986!)a6*-~Itp&02C2P3SvymO!E-^^)$1@+#7Hs!HRxaV+?|LrAD?}L* zSQ)#&WeU@8Mx!szFEr08~k9c`Bgx8l>OP#F}zccu5ifVe&H^W)#d=MnovQ+!k z?2qn9hn^75Z+_TbN{xDjw-kRn@WJ3!^gvoF-*GVaGFD)wj9XVhbbNFo<%~6ZC#O2TQ@;jBA>8c~XZ8_YwH({^zt={rzZ(QD+ zMJ!*=iq@4G#b>vDd~+?D@FY53M%I1@uXnt)eZt%fi}id|jT#I%$@PGR>Aq=l|U6>;@bi_cS6i zeRh{yQV+S<+C%+v@v{o(0peh}C-si}X>*ugfpf4bda>Af34AYoZ~?U+^P<-VcdxwU zGE~A|CsM&FlvG|#O_ay85cl&hkqwocqO;1sP$#(fYNRVYaMqadx2p>m@<1!kRJP1$ zxHk^>LA+EQUg2P3xZ5eFr(7jrkh{Qm2v^2qh5I;uv*jiUC$k;ZBn!8Hx0=71NPaDMf=*24-M93rutIRUnOJ|SJZjHEG5x?KTD?9?uALQlu#R_sUw z@@LoOlIaz}x6z;d&IZ7Jo>dO+K^ge;?XfC8KtA^r-l|pIv=3z*)~9F1POWSfw|4L} zbd)3hSCN!h%XJG=fETi7OV9#^bH@;^+Ao?8-m8(dAGrdDl8(CB{$*AX(?s~jY0 zk2-^y1crEZp(HK0C-^O)*0y>e$K6_xF8D~pFig9j+Y~YzzchJrUIlM3-q+UZ>f&`* z6>l`t*U?JkdL357AN{v)p!L3scZV9@bfs^+wSns$rjAzyy>qQ!H3^k8SuI)U>?&au z!|0cVCswao%1Oi}Lk$yf63AJ2LeV&G9?ky?U8W89x(PR=vGW*=1-WRIoi~EFCCDaN zQFilf1D)zovC68wN1)!xVvTp%IS^~so(S#ZWeM;Fqew0K4gWMy6^*Topb%%btwaJi`mSCljAmFdYnU`+zqKrLmPg@anZ%k zcLl1_dQD*+uzRZkj3lRGE#>LmO6A$eRg{flBNt~3u{Ag3oH^y}D^1*Sztk5O%)KaCK>11cL|+s_b*&!tl>)H zR8evjz4u|7i4RP#qffnZCd(v?yU%tf;56;-n#vsvEcXzzzFEyZdDu)&C!#!D3`)~6 z;QBD1_jY$BzuIyaVZi}1`r zLuqe*`LGyGJaBd;A;CIVIi66)c($OSbf{_u0cet;t1v3$SY9E7@OkW622j#}J1^J? z+K@-rk!-f6gDez7%;e?(DfNE@2SJ;d>AEBv{@B40hB;BoodWpn3jzzkg=5#{*jUZ* z8XGA6;Pgx$(2^vR4p3*HMH>=?cbw8GC|z%PrVVIG$CLD_K`=&~G+{!nECoh4Ma|BE ze`J!xSn6y+g43oLT0vcq3=+pP2xX3N^vz?eJ-`z7ByKy^Tk}kYIJ=Tj#)FU9yKlWU zz7A&A5fZJToIyGVEX31S-GPO42FU|7qcT@LL9^675&(#9h@?_y<&> z3RkJ%+)YQ875EEQ7N<#V)D%>upvF?s6*Z`_a!kblYOEbsv56X+ajMW3M<2uVmBKUk z<Cv+8X|FA18CD7!+K7E7}`qk`H=>F|q!gCM> ziQ3g_aQmjaxjR54RW%=?KJhtMvlBOttBjXq5X)GzGte@*P+<(jRh&s`0S6Hm*9oA{ z`nmcM>TN5xQ6*eoZDo!WUcJp*0n>UMSMU-HNDQl0%vY%Tj5MFVf2PhEOW6v`b z2%q`SnF<6Bs2Not^vcEeCP`o-G6!-bFhu%5r35CLG0-A`i7gx8N?@#N2c{%2CteM# zNnmW*1G17B`_BV=Br#_=1ICgVmv5FcB?$T<{#*@$t|ORxji6(e=h_f-y+3mu2)Y4e zdH_K;LQIb%=tsq;=MZ#L>1iQ?j@>rBiJ+VB$QQz4r?y)94)nBU5E)TQ)cbM*c^F@R z-}DjWn42>vN{U_Cz);*pHBPmP*8@D~q(XmSu~&@p6~y9Y6ujDX2WKzd4@oCpVay{fPJXqb2x|eb*u${9T$?FcJ?c_NuNyfYX6r-}V4%ufp}3o=(v+Akd31kIDk%=@1bVLMVHGIbIGR?ZQM-5W=gUmo>HlBtBfU zb8BwE16sBY_E$w{2;tk_0PpSKqy2S7@&kY&KBt*h@62lB;cjiZcA~yT0$eX?t~7T8be)y&DF~`Kw}^Ef(DfjL zH3+J$3&Z>(pc{z~4j`zhTn5Vv&`q}uZX&4dE|1N90NrBOpf-{^!yS+G=Z#@xC&U_Ycr)9^^5=R|+PG>gIq?=xGR-1=w7pDA9iqDk7D;VF7CQM-DEEft^~8YXHKY5Cnyzf%`q&ydoS_ zI+vLz1A_nd#u_$nA+O__7Q&G1r8ibUC4?M94$H6au4{N zbY2t3$KsF_bzxGce)`IB3XFjReRoCXn4B zxTkwVz5fD%4`_~}CMcDb`fbB)i`I@Ic}bwG?Yrg&!7UO)d56F>T64%za?4h`=o^&k zkX1)v0!Cg#sICr}-hX7MND`1vcLb1cKyU%=-z~fV$CLDusR)R4#{3cjcs>gxQz5ug z+ref6;DqJAbW;Mv{IYosB=9T^Cf|nOrrHNBd;y2t)clPs1)zIF*Y0ikx*wLI2T)lJ z^Bzdx)AN$`6`-9aUbgJo0yZqd!l;DyA&WpDr$ZT1kp@t$h_w<3?!7`>V~R3>+fRtj z!l<=DKfFSK+|heO@ms`7uaWB@IKJPNA|rq*>YN`&0@z;S+UqT22Zq(d0p^r5lcxft dvidi-cGjEgQ}G+Zn)p8=OAn1@ncY6J;D3!l^~eAK literal 0 HcmV?d00001 diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.16.0.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.16.0.jar.sha1 deleted file mode 100644 index 79ed9e0c63fc8..0000000000000 --- a/modules/ingest-geoip/licenses/jackson-annotations-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc30995f7428c0a405eba9b8c619b20d2b3b9905 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.16.1.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..cbc65687606fc --- /dev/null +++ b/modules/ingest-geoip/licenses/jackson-annotations-2.16.1.jar.sha1 @@ -0,0 +1 @@ +fd441d574a71e7d10a4f73de6609f881d8cdfeec \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.16.0.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.16.0.jar.sha1 deleted file mode 100644 index da00d281934b1..0000000000000 --- a/modules/ingest-geoip/licenses/jackson-databind-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3a6b7f8ff7b30d518bbd65678e9c30cd881f19a7 \ No newline at end of file diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.16.1.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..d231db4fd49fc --- /dev/null +++ b/modules/ingest-geoip/licenses/jackson-databind-2.16.1.jar.sha1 @@ -0,0 +1 @@ +02a16efeb840c45af1e2f31753dfe76795278b73 \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/jackson-annotations-2.16.0.jar.sha1 b/plugins/crypto-kms/licenses/jackson-annotations-2.16.0.jar.sha1 deleted file mode 100644 index 79ed9e0c63fc8..0000000000000 --- a/plugins/crypto-kms/licenses/jackson-annotations-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc30995f7428c0a405eba9b8c619b20d2b3b9905 \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/jackson-annotations-2.16.1.jar.sha1 b/plugins/crypto-kms/licenses/jackson-annotations-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..cbc65687606fc --- /dev/null +++ b/plugins/crypto-kms/licenses/jackson-annotations-2.16.1.jar.sha1 @@ -0,0 +1 @@ +fd441d574a71e7d10a4f73de6609f881d8cdfeec \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/jackson-databind-2.16.0.jar.sha1 b/plugins/crypto-kms/licenses/jackson-databind-2.16.0.jar.sha1 deleted file mode 100644 index da00d281934b1..0000000000000 --- a/plugins/crypto-kms/licenses/jackson-databind-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3a6b7f8ff7b30d518bbd65678e9c30cd881f19a7 \ No newline at end of file diff --git a/plugins/crypto-kms/licenses/jackson-databind-2.16.1.jar.sha1 b/plugins/crypto-kms/licenses/jackson-databind-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..d231db4fd49fc --- /dev/null +++ b/plugins/crypto-kms/licenses/jackson-databind-2.16.1.jar.sha1 @@ -0,0 +1 @@ +02a16efeb840c45af1e2f31753dfe76795278b73 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.16.0.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.16.0.jar.sha1 deleted file mode 100644 index 79ed9e0c63fc8..0000000000000 --- a/plugins/discovery-ec2/licenses/jackson-annotations-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc30995f7428c0a405eba9b8c619b20d2b3b9905 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.16.1.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..cbc65687606fc --- /dev/null +++ b/plugins/discovery-ec2/licenses/jackson-annotations-2.16.1.jar.sha1 @@ -0,0 +1 @@ +fd441d574a71e7d10a4f73de6609f881d8cdfeec \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.16.0.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.16.0.jar.sha1 deleted file mode 100644 index da00d281934b1..0000000000000 --- a/plugins/discovery-ec2/licenses/jackson-databind-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3a6b7f8ff7b30d518bbd65678e9c30cd881f19a7 \ No newline at end of file diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.16.1.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..d231db4fd49fc --- /dev/null +++ b/plugins/discovery-ec2/licenses/jackson-databind-2.16.1.jar.sha1 @@ -0,0 +1 @@ +02a16efeb840c45af1e2f31753dfe76795278b73 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-annotations-2.16.0.jar.sha1 b/plugins/repository-azure/licenses/jackson-annotations-2.16.0.jar.sha1 deleted file mode 100644 index 79ed9e0c63fc8..0000000000000 --- a/plugins/repository-azure/licenses/jackson-annotations-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc30995f7428c0a405eba9b8c619b20d2b3b9905 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-annotations-2.16.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-annotations-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..cbc65687606fc --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-annotations-2.16.1.jar.sha1 @@ -0,0 +1 @@ +fd441d574a71e7d10a4f73de6609f881d8cdfeec \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-databind-2.16.0.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.16.0.jar.sha1 deleted file mode 100644 index da00d281934b1..0000000000000 --- a/plugins/repository-azure/licenses/jackson-databind-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3a6b7f8ff7b30d518bbd65678e9c30cd881f19a7 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-databind-2.16.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..d231db4fd49fc --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-databind-2.16.1.jar.sha1 @@ -0,0 +1 @@ +02a16efeb840c45af1e2f31753dfe76795278b73 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.16.0.jar.sha1 b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.16.0.jar.sha1 deleted file mode 100644 index f0d165ff7cf82..0000000000000 --- a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f3cdb002e0f2f30ad9c5fd053d78b1a485511ab1 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-dataformat-xml-2.16.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..ad4e055d4f19a --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-dataformat-xml-2.16.1.jar.sha1 @@ -0,0 +1 @@ +d952ad30d3f2d1220f39db175618414b56d14638 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.16.0.jar.sha1 b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.16.0.jar.sha1 deleted file mode 100644 index 40379694f5ea5..0000000000000 --- a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -77e3a27823f795d928b897d8444744ddb044a5c3 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.16.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..4309dad93b2b6 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-datatype-jsr310-2.16.1.jar.sha1 @@ -0,0 +1 @@ +36a418325c618e440e5ccb80b75c705d894f50bd \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.16.0.jar.sha1 b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.16.0.jar.sha1 deleted file mode 100644 index 820d14b3df8e4..0000000000000 --- a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -684daae9ea45087c670b4f6511edcfdb19c3a695 \ No newline at end of file diff --git a/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.16.1.jar.sha1 b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..5f54d0ac554e0 --- /dev/null +++ b/plugins/repository-azure/licenses/jackson-module-jaxb-annotations-2.16.1.jar.sha1 @@ -0,0 +1 @@ +e9df364a2695e66eb8d2803d6725424842760125 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.16.0.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.16.0.jar.sha1 deleted file mode 100644 index 79ed9e0c63fc8..0000000000000 --- a/plugins/repository-s3/licenses/jackson-annotations-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc30995f7428c0a405eba9b8c619b20d2b3b9905 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.16.1.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..cbc65687606fc --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-annotations-2.16.1.jar.sha1 @@ -0,0 +1 @@ +fd441d574a71e7d10a4f73de6609f881d8cdfeec \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.16.0.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.16.0.jar.sha1 deleted file mode 100644 index da00d281934b1..0000000000000 --- a/plugins/repository-s3/licenses/jackson-databind-2.16.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3a6b7f8ff7b30d518bbd65678e9c30cd881f19a7 \ No newline at end of file diff --git a/plugins/repository-s3/licenses/jackson-databind-2.16.1.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.16.1.jar.sha1 new file mode 100644 index 0000000000000..d231db4fd49fc --- /dev/null +++ b/plugins/repository-s3/licenses/jackson-databind-2.16.1.jar.sha1 @@ -0,0 +1 @@ +02a16efeb840c45af1e2f31753dfe76795278b73 \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/index/mapper/MapperService.java b/server/src/main/java/org/opensearch/index/mapper/MapperService.java index 1e367fb17df36..5cd1e0849a36d 100644 --- a/server/src/main/java/org/opensearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/opensearch/index/mapper/MapperService.java @@ -47,6 +47,7 @@ import org.opensearch.common.settings.Setting.Property; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentContraints; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.core.Assertions; @@ -150,13 +151,45 @@ public enum MergeReason { "index.mapping.depth.limit", 20L, 1, + Integer.MAX_VALUE, + limit -> { + // Make sure XContent constraints are not exceeded (otherwise content processing will fail) + if (limit > XContentContraints.DEFAULT_MAX_DEPTH) { + throw new IllegalArgumentException( + "The provided value " + + limit + + " of the index setting 'index.mapping.depth.limit' exceeds per-JVM configured limit of " + + XContentContraints.DEFAULT_MAX_DEPTH + + ". Please change the setting value or increase per-JVM limit " + + "using '" + + XContentContraints.DEFAULT_MAX_DEPTH_PROPERTY + + "' system property." + ); + } + }, Property.Dynamic, Property.IndexScope ); public static final Setting INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING = Setting.longSetting( "index.mapping.field_name_length.limit", - Long.MAX_VALUE, + Integer.MAX_VALUE, 1L, + Integer.MAX_VALUE, + limit -> { + // Make sure XContent constraints are not exceeded (otherwise content processing will fail) + if (limit > XContentContraints.DEFAULT_MAX_NAME_LEN) { + throw new IllegalArgumentException( + "The provided value " + + limit + + " of the index setting 'index.mapping.field_name_length.limit' exceeds per-JVM configured limit of " + + XContentContraints.DEFAULT_MAX_NAME_LEN + + ". Please change the setting value or increase per-JVM limit " + + "using '" + + XContentContraints.DEFAULT_MAX_NAME_LEN_PROPERTY + + "' system property." + ); + } + }, Property.Dynamic, Property.IndexScope ); diff --git a/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java index f0f34dff0a38f..6ba7ded3e2cb1 100644 --- a/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/MapperServiceTests.java @@ -36,6 +36,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.compress.CompressedXContent; import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.XContentContraints; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.core.common.bytes.BytesReference; import org.opensearch.core.xcontent.XContentBuilder; @@ -158,6 +159,13 @@ public void testMappingDepthExceedsLimit() throws Throwable { assertThat(e.getMessage(), containsString("Limit of mapping depth [1] has been exceeded")); } + public void testMappingDepthXContentLimit() throws Throwable { + createIndex( + "test1", + Settings.builder().put(MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING.getKey(), XContentContraints.DEFAULT_MAX_DEPTH).build() + ); + } + public void testUnmappedFieldType() { MapperService mapperService = createIndex("index").mapperService(); assertThat(mapperService.unmappedFieldType("keyword"), instanceOf(KeywordFieldType.class)); @@ -300,6 +308,15 @@ public void testTotalFieldsLimitWithFieldAlias() throws Throwable { assertEquals("Limit of total fields [" + numberOfNonAliasFields + "] has been exceeded", e.getMessage()); } + public void testFieldNameLengthXContentLimit() throws Throwable { + createIndex( + "test1", + Settings.builder() + .put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), XContentContraints.DEFAULT_MAX_NAME_LEN) + .build() + ); + } + public void testFieldNameLengthLimit() throws Throwable { int maxFieldNameLength = randomIntBetween(25, 30); String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a");