Skip to content

Commit

Permalink
Making fields searchable
Browse files Browse the repository at this point in the history 
Signed-off-by: Harsha Vamsi Kalluri <[email protected]>
  • Loading branch information
@harshavamsi
harshavamsi committed Jan 19, 2024
1 parent e948c40 commit 52c5247
Show file tree
Hide file tree
Showing 5 changed files with 148 additions and 15 deletions.
81 changes: 80 additions & 1 deletion server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,13 @@
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermInSetQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.util.BytesRef;
import org.opensearch.common.Booleans;
Expand Down Expand Up @@ -175,6 +181,10 @@ public BooleanFieldType(String name, boolean searchable) {
this(name, searchable, false, true, false, Collections.emptyMap());
}

public BooleanFieldType(String name, boolean searchable, boolean hasDocValues) {
this(name, searchable, false, hasDocValues, false, Collections.emptyMap());
}

@Override
public String typeName() {
return CONTENT_TYPE;
Expand Down Expand Up @@ -257,9 +267,78 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
return DocValueFormat.BOOLEAN;
}

@Override
public Query termQuery(Object value, QueryShardContext context) {
failIfNotIndexedAndNoDocValues();
Query query = new TermQuery(new Term(name(), indexedValueForSearch(value)));
if (boost() != 1f) {
query = new BoostQuery(query, boost());
}
if (isSearchable() && hasDocValues()) {
return new IndexOrDocValuesQuery(
query,
SortedNumericDocValuesField.newSlowExactQuery(name(), Values.TRUE.bytesEquals(indexedValueForSearch(value)) ? 1 : 0)
);
}
if (hasDocValues()) {
return SortedNumericDocValuesField.newSlowExactQuery(name(), Values.TRUE.bytesEquals(indexedValueForSearch(value)) ? 1 : 0);
}
return query;
}

@Override
public Query termsQuery(List<?> values, QueryShardContext context) {
failIfNotIndexedAndNoDocValues();
if (isSearchable() && hasDocValues()) {
Query query = new TermInSetQuery(name(), values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new));
Query dvQuery = new TermInSetQuery(
MultiTermQuery.DOC_VALUES_REWRITE,
name(),
values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new)
);
return new IndexOrDocValuesQuery(query, dvQuery);
}
if (hasDocValues()) {
return new TermInSetQuery(
MultiTermQuery.DOC_VALUES_REWRITE,
name(),
values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new)
);
}
return new TermInSetQuery(name(), values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new));
}

@Override
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
failIfNotIndexed();
failIfNotIndexedAndNoDocValues();
if (isSearchable() && hasDocValues()) {
Query query = new TermRangeQuery(
name(),
lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
upperTerm == null ? null : indexedValueForSearch(upperTerm),
includeLower,
includeUpper
);
Query dvQuery = new TermRangeQuery(
name(),
lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
upperTerm == null ? null : indexedValueForSearch(upperTerm),
includeLower,
includeUpper,
MultiTermQuery.DOC_VALUES_REWRITE
);
return new IndexOrDocValuesQuery(query, dvQuery);
}
if (hasDocValues()) {
return new TermRangeQuery(
name(),
lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
upperTerm == null ? null : indexedValueForSearch(upperTerm),
includeLower,
includeUpper,
MultiTermQuery.DOC_VALUES_REWRITE
);
}
return new TermRangeQuery(
name(),
lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
Expand Down
19 changes: 14 additions & 5 deletions server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,22 +457,30 @@ public Query rangeQuery(
@Nullable DateMathParser forcedDateParser,
QueryShardContext context
) {
failIfNotIndexed();
failIfNotIndexedAndNoDocValues();
if (relation == ShapeRelation.DISJOINT) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges");
}
DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser;
return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> {
if(isSearchable() && hasDocValues()){
Query query = LongPoint.newRangeQuery(name(), l, u);
if (hasDocValues()) {
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u);
query = new IndexOrDocValuesQuery(query, dvQuery);
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u);
query = new IndexOrDocValuesQuery(query, dvQuery);

if (context.indexSortedOnField(name())) {
query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query);
}
return query;
}
if (hasDocValues()) {
Query query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u);
if (context.indexSortedOnField(name())) {
query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query);
}
return query;
}
return query;
return LongPoint.newRangeQuery(name(), l, u);
});
}

Expand Down Expand Up @@ -543,6 +551,7 @@ public static long parseToLong(

@Override
public Query distanceFeatureQuery(Object origin, String pivot, float boost, QueryShardContext context) {
failIfNotIndexedAndNoDocValues();
long originLong = parseToLong(origin, true, null, null, context::nowInMillis);
TimeValue pivotTime = TimeValue.parseTimeValue(pivot, "distance_feature.pivot");
return resolution.distanceFeatureQuery(name(), boost, originLong, pivotTime);
Expand Down
7 changes: 6 additions & 1 deletion server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,16 @@

package org.opensearch.index.mapper;

import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.opensearch.common.xcontent.XContentFactory;
Expand All @@ -46,6 +50,7 @@
import org.opensearch.index.mapper.ParseContext.Document;

import java.io.IOException;
import java.util.SortedSet;

public class BooleanFieldMapperTests extends MapperTestCase {

Expand Down Expand Up @@ -206,7 +211,7 @@ public void testBoosts() throws Exception {
}));

MappedFieldType ft = mapperService.fieldType("field");
assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2.0f), ft.termQuery("true", null));
assertEquals(new IndexOrDocValuesQuery(new BoostQuery(new TermQuery(new Term("field", "T")), 2.0f), SortedNumericDocValuesField.newSlowExactQuery("field", 1)), ft.termQuery("true", null));
assertParseMaximalWarnings();
}
}
48 changes: 44 additions & 4 deletions server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,18 @@

package org.opensearch.index.mapper;

import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.TermInSetQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

public class BooleanFieldTypeTests extends FieldTypeTestCase {

Expand All @@ -56,12 +63,45 @@ public void testValueForSearch() {

public void testTermQuery() {
MappedFieldType ft = new BooleanFieldMapper.BooleanFieldType("field");
assertEquals(new TermQuery(new Term("field", "T")), ft.termQuery("true", null));
assertEquals(new TermQuery(new Term("field", "F")), ft.termQuery("false", null));
assertEquals(
new IndexOrDocValuesQuery(new TermQuery(new Term("field", "T")), SortedNumericDocValuesField.newSlowExactQuery("field", 1)),
ft.termQuery("true", null)
);
assertEquals(
new IndexOrDocValuesQuery(new TermQuery(new Term("field", "F")), SortedNumericDocValuesField.newSlowExactQuery("field", 0)),
ft.termQuery("false", null)
);

MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false);
MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("true", null));
assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage());
}

public void testTermsQuery() {
MappedFieldType ft = new BooleanFieldMapper.BooleanFieldType("field");
BooleanFieldMapper.BooleanFieldType booleanFieldType = new BooleanFieldMapper.BooleanFieldType("field");
List<BytesRef> terms = new ArrayList<>();
terms.add(new BytesRef("true"));
terms.add(new BytesRef("false"));
assertEquals(
new IndexOrDocValuesQuery(
new TermInSetQuery("field", terms.stream().map(booleanFieldType::indexedValueForSearch).toArray(BytesRef[]::new)),
new TermInSetQuery(
MultiTermQuery.DOC_VALUES_REWRITE,
"field",
terms.stream().map(booleanFieldType::indexedValueForSearch).toArray(BytesRef[]::new)
)
),
ft.termsQuery(terms, null)
);
assertEquals(new IndexOrDocValuesQuery(
new TermInSetQuery("field", terms),
new TermInSetQuery(MultiTermQuery.DOC_VALUES_REWRITE, "field", terms)
), ft.termsQuery(terms, null));

MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termsQuery(terms, null));
assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage());
}

public void testFetchSourceValue() throws IOException {
Expand Down
8 changes: 4 additions & 4 deletions server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,14 +216,14 @@ public void testTermQuery() {
"field",
false,
false,
true,
false,
DateFieldMapper.getDefaultDateTimeFormatter(),
Resolution.MILLISECONDS,
null,
Collections.emptyMap()
);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery(date, context));
assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage());
}

public void testRangeQuery() throws IOException {
Expand Down Expand Up @@ -279,7 +279,7 @@ public void testRangeQuery() throws IOException {
"field",
false,
false,
true,
false,
DateFieldMapper.getDefaultDateTimeFormatter(),
Resolution.MILLISECONDS,
null,
Expand All @@ -289,7 +289,7 @@ public void testRangeQuery() throws IOException {
IllegalArgumentException.class,
() -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context)
);
assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage());
}

public void testRangeQueryWithIndexSort() {
Expand Down

0 comments on commit 52c5247

Please sign in to comment.