diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml index a133060f07c6f..c23584102b212 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml @@ -71,6 +71,166 @@ - '{ "index": { "_index": "test-iodvq", "_id": "3" } }' - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.987654321Z", "date": "2024-10-29T12:12:12.987Z" }' + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + terms: + some_keyword: ["400", "5"] + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + terms: { + some_keyword: ["400", "5"], + rewrite_override: "index_only" + } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + terms: { + some_keyword: ["400", "5"], + rewrite_override: "doc_values_only" + } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + fuzzy: + some_keyword: { + value: "402" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + fuzzy: + some_keyword: { + value: "402", + rewrite_override: "index_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + fuzzy: + some_keyword: { + value: "402", + rewrite_override: "doc_values_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + regexp: + some_keyword: { + value: "40*" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + regexp: + some_keyword: { + value: "40*", + rewrite_override: "index_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + regexp: + some_keyword: { + value: "40*", + rewrite_override: "doc_values_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + wildcard: + some_keyword: { + value: "ing*" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + wildcard: + some_keyword: { + value: "ing*", + rewrite_override: "index_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + wildcard: + some_keyword: { + value: "ing*", + rewrite_override: "doc_values_only" + } + + - match: { hits.total: 1 } + - do: search: rest_total_hits_as_int: true @@ -82,6 +242,34 @@ - match: { hits.hits.0._source.some_keyword: "ingesting some random keyword data" } + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + prefix: + some_keyword: { + value: "ing", + rewrite_override: "index_only" + } + + - match: { hits.hits.0._source.some_keyword: "ingesting some random keyword data" } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + prefix: + some_keyword: { + value: "ing", + rewrite_override: "doc_values_only" + } + + - match: { hits.hits.0._source.some_keyword: "ingesting some random keyword data" } + - do: search: rest_total_hits_as_int: true @@ -95,6 +283,36 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + range: { + "some_keyword": { + "lt": 500, + rewrite_override: "index_only" + + } } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-iodvq + body: + query: + range: { + "some_keyword": { + "lt": 500, + rewrite_override: "doc_values_only" + + } } + + - match: { hits.total: 2 } + - do: search: @@ -608,6 +826,111 @@ - '{ "index": { "_index": "test-index", "_id": "3" } }' - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.123456789Z", "date": "2024-10-29T12:12:12.987Z" }' + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + terms: + some_keyword: ["400", "5"] + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + terms: { + some_keyword: ["400", "5"], + rewrite_override: "index_only" + } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + fuzzy: + some_keyword: { + value: "402" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + fuzzy: + some_keyword: { + value: "402", + rewrite_override: "index_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + regexp: + some_keyword: { + value: "40*" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + regexp: + some_keyword: { + value: "40*", + rewrite_override: "index_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + wildcard: + some_keyword: { + value: "ing*" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + wildcard: + some_keyword: { + value: "ing*", + rewrite_override: "index_only" + } + + - match: { hits.total: 1 } + - do: search: rest_total_hits_as_int: true @@ -619,6 +942,20 @@ - match: { hits.hits.0._source.some_keyword: "ingesting some random keyword data" } + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + prefix: + some_keyword: { + value: "ing", + rewrite_override: "index_only" + } + + - match: { hits.hits.0._source.some_keyword: "ingesting some random keyword data" } + - do: search: rest_total_hits_as_int: true @@ -632,6 +969,21 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-index + body: + query: + range: { + "some_keyword": { + "lt": 500, + rewrite_override: "index_only" + + } } + + - match: { hits.total: 2 } + - do: search: rest_total_hits_as_int: true @@ -1150,6 +1502,111 @@ - '{ "index": { "_index": "test-doc-values", "_id": "3" } }' - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.123456789Z", "date": "2024-10-29T12:12:12.987Z" }' + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + terms: + some_keyword: ["400", "5"] + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + terms: { + some_keyword: ["400", "5"], + rewrite_override: "doc_values_only" + } + + - match: { hits.total: 2 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + fuzzy: + some_keyword: { + value: "402" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + fuzzy: + some_keyword: { + value: "402", + rewrite_override: "doc_values_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + regexp: + some_keyword: { + value: "40*" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + regexp: + some_keyword: { + value: "40*", + rewrite_override: "doc_values_only" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + wildcard: + some_keyword: { + value: "ing*" + } + + - match: { hits.total: 1 } + + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + wildcard: + some_keyword: { + value: "ing*", + rewrite_override: "doc_values_only" + } + + - match: { hits.total: 1 } + - do: search: rest_total_hits_as_int: true @@ -1161,6 +1618,20 @@ - match: { hits.hits.0._source.some_keyword: "ingesting some random keyword data" } + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + prefix: + some_keyword: { + value: "ing", + rewrite_override: "doc_values_only" + } + + - match: { hits.hits.0._source.some_keyword: "ingesting some random keyword data" } + - do: search: rest_total_hits_as_int: true @@ -1174,6 +1645,21 @@ - match: { hits.total: 2 } + - do: + search: + rest_total_hits_as_int: true + index: test-doc-values + body: + query: + range: { + "some_keyword": { + "lt": 500, + rewrite_override: "doc_values_only" + + } } + + - match: { hits.total: 2 } + - do: search: rest_total_hits_as_int: true diff --git a/server/src/main/java/org/opensearch/index/mapper/SimpleMappedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/SimpleMappedFieldType.java index 70c0da3e42dd6..3616458e8d123 100644 --- a/server/src/main/java/org/opensearch/index/mapper/SimpleMappedFieldType.java +++ b/server/src/main/java/org/opensearch/index/mapper/SimpleMappedFieldType.java @@ -33,6 +33,7 @@ package org.opensearch.index.mapper; import org.apache.lucene.search.Query; +import org.opensearch.common.Nullable; import org.opensearch.common.geo.ShapeRelation; import org.opensearch.common.time.DateMathParser; import org.opensearch.index.query.QueryShardContext; @@ -78,6 +79,27 @@ public final Query rangeQuery( return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); } + @Override + public final Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + ShapeRelation relation, + ZoneId timeZone, + DateMathParser parser, + @Nullable RewriteOverride rewriteOverride, + QueryShardContext context + ) { + if (relation == ShapeRelation.DISJOINT) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges"); + } + // We do not fail on non-null time zones and date parsers + // The reasoning is that on query parsers, you might want to set a time zone or format for date fields + // but then the API has no way to know which fields are dates and which fields are not dates + return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, rewriteOverride, context); + } + /** * Same as {@link #rangeQuery(Object, Object, boolean, boolean, ShapeRelation, ZoneId, DateMathParser, QueryShardContext)} * but without the trouble of relations or date-specific options. @@ -94,7 +116,7 @@ protected Query rangeQuery( RewriteOverride rewriteOverride, QueryShardContext context ) { - throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support range queries"); + return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); } } diff --git a/server/src/main/java/org/opensearch/index/mapper/StringFieldType.java b/server/src/main/java/org/opensearch/index/mapper/StringFieldType.java index 682ccc13f769d..08831b3d151f5 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StringFieldType.java +++ b/server/src/main/java/org/opensearch/index/mapper/StringFieldType.java @@ -45,6 +45,7 @@ import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.automaton.Operations; import org.opensearch.OpenSearchException; +import org.opensearch.common.Nullable; import org.opensearch.common.lucene.BytesRefs; import org.opensearch.common.lucene.search.AutomatonQueries; import org.opensearch.common.unit.Fuzziness; @@ -110,7 +111,7 @@ public Query fuzzyQuery( int prefixLength, int maxExpansions, boolean transpositions, - MultiTermQuery.RewriteMethod method, + @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context ) { if (!context.allowExpensiveQueries()) { @@ -132,6 +133,20 @@ public Query fuzzyQuery( ); } + @Override + public Query fuzzyQuery( + Object value, + Fuzziness fuzziness, + int prefixLength, + int maxExpansions, + boolean transpositions, + @Nullable MultiTermQuery.RewriteMethod method, + @Nullable RewriteOverride rewriteOverride, + QueryShardContext context + ) { + return fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, method, context); + } + @Override public Query prefixQuery(String value, MultiTermQuery.RewriteMethod method, boolean caseInsensitive, QueryShardContext context) { if (context.allowExpensiveQueries() == false) { @@ -188,6 +203,17 @@ public Query wildcardQuery(String value, MultiTermQuery.RewriteMethod method, bo return wildcardQuery(value, method, caseInsensitive, false, context); } + @Override + public Query wildcardQuery( + String value, + MultiTermQuery.RewriteMethod method, + @Nullable RewriteOverride rewriteOverride, + boolean caseInsensitive, + QueryShardContext context + ) { + return wildcardQuery(value, method, caseInsensitive, context); + } + /** always normalizes the wildcard pattern to lowercase */ @Override public Query normalizedWildcardQuery(String value, MultiTermQuery.RewriteMethod method, QueryShardContext context) { @@ -279,4 +305,16 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower includeUpper ); } + + @Override + public Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + @Nullable RewriteOverride rewriteOverride, + QueryShardContext context + ) { + return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); + } } diff --git a/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java index acf3fac459267..cf774fef0e88c 100644 --- a/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/FuzzyQueryBuilder.java @@ -381,14 +381,14 @@ protected Query doToQuery(QueryShardContext context) throws IOException { throw new IllegalStateException("Rewrite first"); } String rewrite = this.rewrite; - Query query = fieldType.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, null, context); + RewriteOverride rewriteOverride = QueryParsers.parseRewriteOverride( + rewrite_override, + RewriteOverride.DEFAULT, + LoggingDeprecationHandler.INSTANCE + ); + Query query = fieldType.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, null, rewriteOverride, context); if (query instanceof MultiTermQuery) { MultiTermQuery.RewriteMethod rewriteMethod = QueryParsers.parseRewriteMethod(rewrite, null, LoggingDeprecationHandler.INSTANCE); - RewriteOverride rewriteOverride = QueryParsers.parseRewriteOverride( - rewrite_override, - RewriteOverride.DEFAULT, - LoggingDeprecationHandler.INSTANCE - ); query = fieldType.fuzzyQuery( value, fuzziness, diff --git a/server/src/main/java/org/opensearch/index/query/PrefixQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/PrefixQueryBuilder.java index b40b0250f4029..2e6212f5b0948 100644 --- a/server/src/main/java/org/opensearch/index/query/PrefixQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/PrefixQueryBuilder.java @@ -270,7 +270,12 @@ protected Query doToQuery(QueryShardContext context) throws IOException { if (fieldType == null) { throw new IllegalStateException("Rewrite first"); } - return fieldType.prefixQuery(value, method, caseInsensitive, context); + RewriteOverride rewriteOverride = QueryParsers.parseRewriteOverride( + rewrite_override, + RewriteOverride.DEFAULT, + LoggingDeprecationHandler.INSTANCE + ); + return fieldType.prefixQuery(value, method, rewriteOverride, caseInsensitive, context); } @Override diff --git a/server/src/main/java/org/opensearch/index/query/WildcardQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/WildcardQueryBuilder.java index 3916eeb41e426..f905f6bb389f8 100644 --- a/server/src/main/java/org/opensearch/index/query/WildcardQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/query/WildcardQueryBuilder.java @@ -282,7 +282,12 @@ protected Query doToQuery(QueryShardContext context) throws IOException { } MultiTermQuery.RewriteMethod method = QueryParsers.parseRewriteMethod(rewrite, null, LoggingDeprecationHandler.INSTANCE); - return fieldType.wildcardQuery(value, method, caseInsensitive, context); + RewriteOverride rewriteOverride = QueryParsers.parseRewriteOverride( + rewrite_override, + RewriteOverride.DEFAULT, + LoggingDeprecationHandler.INSTANCE + ); + return fieldType.wildcardQuery(value, method, rewriteOverride, caseInsensitive, context); } @Override diff --git a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java index 86f645957b78b..23b56a5fa4e87 100644 --- a/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/RangeQueryBuilderTests.java @@ -420,7 +420,7 @@ public void testFromJson() throws IOException { + " \"include_lower\" : true,\n" + " \"include_upper\" : true,\n" + " \"time_zone\" : \"+01:00\",\n" - + " \"boost\" : 1.0\n" + + " \"boost\" : 1.0,\n" + " \"rewrite_override\" : \"index_only\"\n" + " }\n" + " }\n" diff --git a/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java index 80c87b5e85e30..f8140715cf566 100644 --- a/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/query/RegexpQueryBuilderTests.java @@ -122,7 +122,7 @@ public void testFromJson() throws IOException { + " \"flags_value\" : 7,\n" + " \"case_insensitive\" : true,\n" + " \"max_determinized_states\" : 20000,\n" - + " \"boost\" : 1.0\n" + + " \"boost\" : 1.0,\n" + " \"rewrite_override\" : \"index_only\"\n" + " }\n" + " }\n"