From 52c524723c521cd3438b80b9221fad810d7476ff Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Thu, 18 Jan 2024 21:21:17 -0800 Subject: [PATCH] Making fields searchable Signed-off-by: Harsha Vamsi Kalluri --- .../index/mapper/BooleanFieldMapper.java | 81 ++++++++++++++++++- .../index/mapper/DateFieldMapper.java | 19 +++-- .../index/mapper/BooleanFieldMapperTests.java | 7 +- .../index/mapper/BooleanFieldTypeTests.java | 48 ++++++++++- .../index/mapper/DateFieldTypeTests.java | 8 +- 5 files changed, 148 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java index 3c7925809415a..cedd4de5343bb 100644 --- a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java @@ -37,7 +37,13 @@ import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermInSetQuery; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; import org.opensearch.common.Booleans; @@ -175,6 +181,10 @@ public BooleanFieldType(String name, boolean searchable) { this(name, searchable, false, true, false, Collections.emptyMap()); } + public BooleanFieldType(String name, boolean searchable, boolean hasDocValues) { + this(name, searchable, false, hasDocValues, false, Collections.emptyMap()); + } + @Override public String typeName() { return CONTENT_TYPE; @@ -257,9 +267,78 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { return DocValueFormat.BOOLEAN; } + @Override + public Query termQuery(Object value, QueryShardContext context) { + failIfNotIndexedAndNoDocValues(); + Query query = new TermQuery(new Term(name(), indexedValueForSearch(value))); + if (boost() != 1f) { + query = new BoostQuery(query, boost()); + } + if (isSearchable() && hasDocValues()) { + return new IndexOrDocValuesQuery( + query, + SortedNumericDocValuesField.newSlowExactQuery(name(), Values.TRUE.bytesEquals(indexedValueForSearch(value)) ? 1 : 0) + ); + } + if (hasDocValues()) { + return SortedNumericDocValuesField.newSlowExactQuery(name(), Values.TRUE.bytesEquals(indexedValueForSearch(value)) ? 1 : 0); + } + return query; + } + + @Override + public Query termsQuery(List values, QueryShardContext context) { + failIfNotIndexedAndNoDocValues(); + if (isSearchable() && hasDocValues()) { + Query query = new TermInSetQuery(name(), values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new)); + Query dvQuery = new TermInSetQuery( + MultiTermQuery.DOC_VALUES_REWRITE, + name(), + values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new) + ); + return new IndexOrDocValuesQuery(query, dvQuery); + } + if (hasDocValues()) { + return new TermInSetQuery( + MultiTermQuery.DOC_VALUES_REWRITE, + name(), + values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new) + ); + } + return new TermInSetQuery(name(), values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new)); + } + @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { - failIfNotIndexed(); + failIfNotIndexedAndNoDocValues(); + if (isSearchable() && hasDocValues()) { + Query query = new TermRangeQuery( + name(), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), + includeLower, + includeUpper + ); + Query dvQuery = new TermRangeQuery( + name(), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), + includeLower, + includeUpper, + MultiTermQuery.DOC_VALUES_REWRITE + ); + return new IndexOrDocValuesQuery(query, dvQuery); + } + if (hasDocValues()) { + return new TermRangeQuery( + name(), + lowerTerm == null ? null : indexedValueForSearch(lowerTerm), + upperTerm == null ? null : indexedValueForSearch(upperTerm), + includeLower, + includeUpper, + MultiTermQuery.DOC_VALUES_REWRITE + ); + } return new TermRangeQuery( name(), lowerTerm == null ? null : indexedValueForSearch(lowerTerm), diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java index d98e6ea6af83d..d03476738c9cc 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java @@ -457,22 +457,30 @@ public Query rangeQuery( @Nullable DateMathParser forcedDateParser, QueryShardContext context ) { - failIfNotIndexed(); + failIfNotIndexedAndNoDocValues(); if (relation == ShapeRelation.DISJOINT) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges"); } DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser; return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { + if(isSearchable() && hasDocValues()){ Query query = LongPoint.newRangeQuery(name(), l, u); - if (hasDocValues()) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); - query = new IndexOrDocValuesQuery(query, dvQuery); + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + if (context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } + return query; + } + if (hasDocValues()) { + Query query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); if (context.indexSortedOnField(name())) { query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); } + return query; } - return query; + return LongPoint.newRangeQuery(name(), l, u); }); } @@ -543,6 +551,7 @@ public static long parseToLong( @Override public Query distanceFeatureQuery(Object origin, String pivot, float boost, QueryShardContext context) { + failIfNotIndexedAndNoDocValues(); long originLong = parseToLong(origin, true, null, null, context::nowInMillis); TimeValue pivotTime = TimeValue.parseTimeValue(pivot, "distance_feature.pivot"); return resolution.distanceFeatureQuery(name(), boost, originLong, pivotTime); diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java index 8dec03a353d16..6981b1dd544be 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java @@ -32,12 +32,16 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Term; import org.apache.lucene.search.BoostQuery; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.opensearch.common.xcontent.XContentFactory; @@ -46,6 +50,7 @@ import org.opensearch.index.mapper.ParseContext.Document; import java.io.IOException; +import java.util.SortedSet; public class BooleanFieldMapperTests extends MapperTestCase { @@ -206,7 +211,7 @@ public void testBoosts() throws Exception { })); MappedFieldType ft = mapperService.fieldType("field"); - assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2.0f), ft.termQuery("true", null)); + assertEquals(new IndexOrDocValuesQuery(new BoostQuery(new TermQuery(new Term("field", "T")), 2.0f), SortedNumericDocValuesField.newSlowExactQuery("field", 1)), ft.termQuery("true", null)); assertParseMaximalWarnings(); } } diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java index 14092706411cb..0f843b86988bd 100644 --- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java @@ -31,11 +31,18 @@ package org.opensearch.index.mapper; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; public class BooleanFieldTypeTests extends FieldTypeTestCase { @@ -56,12 +63,45 @@ public void testValueForSearch() { public void testTermQuery() { MappedFieldType ft = new BooleanFieldMapper.BooleanFieldType("field"); - assertEquals(new TermQuery(new Term("field", "T")), ft.termQuery("true", null)); - assertEquals(new TermQuery(new Term("field", "F")), ft.termQuery("false", null)); + assertEquals( + new IndexOrDocValuesQuery(new TermQuery(new Term("field", "T")), SortedNumericDocValuesField.newSlowExactQuery("field", 1)), + ft.termQuery("true", null) + ); + assertEquals( + new IndexOrDocValuesQuery(new TermQuery(new Term("field", "F")), SortedNumericDocValuesField.newSlowExactQuery("field", 0)), + ft.termQuery("false", null) + ); - MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false); + MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("true", null)); - assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); + } + + public void testTermsQuery() { + MappedFieldType ft = new BooleanFieldMapper.BooleanFieldType("field"); + BooleanFieldMapper.BooleanFieldType booleanFieldType = new BooleanFieldMapper.BooleanFieldType("field"); + List terms = new ArrayList<>(); + terms.add(new BytesRef("true")); + terms.add(new BytesRef("false")); + assertEquals( + new IndexOrDocValuesQuery( + new TermInSetQuery("field", terms.stream().map(booleanFieldType::indexedValueForSearch).toArray(BytesRef[]::new)), + new TermInSetQuery( + MultiTermQuery.DOC_VALUES_REWRITE, + "field", + terms.stream().map(booleanFieldType::indexedValueForSearch).toArray(BytesRef[]::new) + ) + ), + ft.termsQuery(terms, null) + ); + assertEquals(new IndexOrDocValuesQuery( + new TermInSetQuery("field", terms), + new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, "field", terms) + ), ft.termsQuery(terms, null)); + + MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termsQuery(terms, null)); + assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); } public void testFetchSourceValue() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java index ab53ae81ab0ce..db5e1e419de93 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java @@ -216,14 +216,14 @@ public void testTermQuery() { "field", false, false, - true, + false, DateFieldMapper.getDefaultDateTimeFormatter(), Resolution.MILLISECONDS, null, Collections.emptyMap() ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery(date, context)); - assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); } public void testRangeQuery() throws IOException { @@ -279,7 +279,7 @@ public void testRangeQuery() throws IOException { "field", false, false, - true, + false, DateFieldMapper.getDefaultDateTimeFormatter(), Resolution.MILLISECONDS, null, @@ -289,7 +289,7 @@ public void testRangeQuery() throws IOException { IllegalArgumentException.class, () -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context) ); - assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); + assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage()); } public void testRangeQueryWithIndexSort() {