Skip to content

Commit

Permalink
Fix tests for keyword terms
Browse files Browse the repository at this point in the history 
Signed-off-by: Harsha Vamsi Kalluri <[email protected]>
  • Loading branch information
@harshavamsi
harshavamsi committed Dec 21, 2023
1 parent 31d32b8 commit a7f6bd2
Show file tree
Hide file tree
Showing 12 changed files with 99 additions and 64 deletions.
10 changes: 8 additions & 2 deletions server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,15 @@
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.FuzzyQuery;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.RegexpQuery;
import org.apache.lucene.search.TermInSetQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -390,16 +392,20 @@ protected BytesRef indexedValueForSearch(Object value) {
@Override
public Query termQuery(Object value, QueryShardContext context) {
failIfNotIndexedAndNoDocValues();
Query query = new TermQuery(new Term(name(), indexedValueForSearch(value)));
if (boost() != 1f) {
query = new BoostQuery(query, boost());
}
if (isSearchable() && hasDocValues()) {
return new IndexOrDocValuesQuery(
super.termQuery(value, context),
query,
SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value))
);
}
if (hasDocValues()) {
return SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value));
}
return super.termQuery(value, context);
return query;
}

@Override
Expand Down
2 changes: 2 additions & 0 deletions server/src/main/java/org/opensearch/index/query/QueryBuilders.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@

package org.opensearch.index.query;

import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.join.ScoreMode;
import org.opensearch.common.Nullable;
import org.opensearch.common.geo.GeoPoint;
Expand Down
6 changes: 3 additions & 3 deletions server/src/test/java/org/opensearch/index/mapper/KeywordFieldTypeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ protected TokenStream normalize(String fieldName, TokenStream in) {
}
};
MappedFieldType ft = new KeywordFieldType("field", new NamedAnalyzer("my_normalizer", AnalyzerScope.INDEX, normalizer));
assertEquals(new TermQuery(new Term("field", "foo bar")), ft.termQuery("fOo BaR", null));
assertEquals(new IndexOrDocValuesQuery(new TermQuery(new Term("field", "foo bar")), SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef("foo bar"))), ft.termQuery("fOo BaR", null));
}

public void testTermsQuery() {
Expand Down Expand Up @@ -345,9 +345,9 @@ public void testWildCardQuery() {

public void testNormalizeQueries() {
MappedFieldType ft = new KeywordFieldType("field");
assertEquals(new TermQuery(new Term("field", new BytesRef("FOO"))), ft.termQuery("FOO", null));
assertEquals(new IndexOrDocValuesQuery(new TermQuery(new Term("field", new BytesRef("FOO"))), SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef("FOO"))), ft.termQuery("FOO", null));
ft = new KeywordFieldType("field", Lucene.STANDARD_ANALYZER);
assertEquals(new TermQuery(new Term("field", new BytesRef("foo"))), ft.termQuery("FOO", null));
assertEquals(new IndexOrDocValuesQuery(new TermQuery(new Term("field", new BytesRef("foo"))), SortedSetDocValuesField.newSlowExactQuery("field", new BytesRef("foo"))), ft.termQuery("FOO", null));
}

public void testFetchSourceValue() throws IOException {
Expand Down
18 changes: 10 additions & 8 deletions server/src/test/java/org/opensearch/index/query/MultiMatchQueryBuilderTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

package org.opensearch.index.query;

import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.ExtendedCommonTermsQuery;
import org.apache.lucene.search.BooleanClause;
Expand All @@ -47,6 +48,7 @@
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery;
import org.opensearch.common.settings.Settings;
Expand Down Expand Up @@ -242,7 +244,7 @@ public void testToQueryMultipleFieldsDisableDismax() throws Exception {
.tieBreaker(1.0f)
.toQuery(createShardContext());
Query expected = new DisjunctionMaxQuery(
List.of(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))),
List.of(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new IndexOrDocValuesQuery(new TermQuery(new Term(KEYWORD_FIELD_NAME, "test")), SortedSetDocValuesField.newSlowExactQuery(KEYWORD_FIELD_NAME, new BytesRef("test")))),
1
);
assertEquals(expected, query);
Expand All @@ -251,7 +253,7 @@ public void testToQueryMultipleFieldsDisableDismax() throws Exception {
public void testToQueryMultipleFieldsDisMaxQuery() throws Exception {
Query query = multiMatchQuery("test").field(TEXT_FIELD_NAME).field(KEYWORD_FIELD_NAME).toQuery(createShardContext());
Query expected = new DisjunctionMaxQuery(
List.of(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))),
List.of(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new IndexOrDocValuesQuery(new TermQuery(new Term(KEYWORD_FIELD_NAME, "test")), SortedSetDocValuesField.newSlowExactQuery(KEYWORD_FIELD_NAME, new BytesRef("test")))),
0
);
assertEquals(expected, query);
Expand All @@ -260,7 +262,7 @@ public void testToQueryMultipleFieldsDisMaxQuery() throws Exception {
public void testToQueryFieldsWildcard() throws Exception {
Query query = multiMatchQuery("test").field("mapped_str*").tieBreaker(1.0f).toQuery(createShardContext());
Query expected = new DisjunctionMaxQuery(
List.of(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "test"))),
List.of(new TermQuery(new Term(TEXT_FIELD_NAME, "test")), new IndexOrDocValuesQuery(new TermQuery(new Term(KEYWORD_FIELD_NAME, "test")), SortedSetDocValuesField.newSlowExactQuery(KEYWORD_FIELD_NAME, new BytesRef("test")))),
1
);
assertEquals(expected, query);
Expand Down Expand Up @@ -459,7 +461,7 @@ public void testDefaultField() throws Exception {
DisjunctionMaxQuery expected = new DisjunctionMaxQuery(
Arrays.asList(
new TermQuery(new Term(TEXT_FIELD_NAME, "hello")),
new BoostQuery(new TermQuery(new Term(KEYWORD_FIELD_NAME, "hello")), 5.0f)
new BoostQuery(new IndexOrDocValuesQuery(new TermQuery(new Term(KEYWORD_FIELD_NAME, "hello")), SortedSetDocValuesField.newSlowExactQuery(KEYWORD_FIELD_NAME, new BytesRef("hello"))), 5.0f)
),
0.0f
);
Expand Down Expand Up @@ -487,7 +489,7 @@ public void testDefaultField() throws Exception {
Arrays.asList(
new MatchNoDocsQuery("failed [mapped_int] query, caused by number_format_exception:[For input string: \"hello\"]"),
new TermQuery(new Term(TEXT_FIELD_NAME, "hello")),
new BoostQuery(new TermQuery(new Term(KEYWORD_FIELD_NAME, "hello")), 5.0f)
new BoostQuery(new IndexOrDocValuesQuery(new TermQuery(new Term(KEYWORD_FIELD_NAME, "hello")), SortedSetDocValuesField.newSlowExactQuery(KEYWORD_FIELD_NAME, new BytesRef("hello"))), 5.0f)
),
0.0f
);
Expand Down Expand Up @@ -531,8 +533,8 @@ public void testWithStopWords() throws Exception {
new BooleanQuery.Builder().add(new TermQuery(new Term(TEXT_FIELD_NAME, "quick")), BooleanClause.Occur.SHOULD)
.add(new TermQuery(new Term(TEXT_FIELD_NAME, "fox")), BooleanClause.Occur.SHOULD)
.build(),
new BooleanQuery.Builder().add(new TermQuery(new Term(KEYWORD_FIELD_NAME, "quick")), BooleanClause.Occur.SHOULD)
.add(new TermQuery(new Term(KEYWORD_FIELD_NAME, "fox")), BooleanClause.Occur.SHOULD)
new BooleanQuery.Builder().add(new IndexOrDocValuesQuery(new TermQuery(new Term(KEYWORD_FIELD_NAME, "quick")), SortedSetDocValuesField.newSlowExactQuery(KEYWORD_FIELD_NAME, new BytesRef("quick"))), BooleanClause.Occur.SHOULD)
.add(new IndexOrDocValuesQuery(new TermQuery(new Term(KEYWORD_FIELD_NAME, "fox")), SortedSetDocValuesField.newSlowExactQuery(KEYWORD_FIELD_NAME, new BytesRef("fox"))), BooleanClause.Occur.SHOULD)
.build()
),
0f
Expand Down Expand Up @@ -588,7 +590,7 @@ private void assertQueryWithAllFieldsWildcard(Query query) {
assertEquals(9, noMatchNoDocsQueries);
assertThat(
disjunctionMaxQuery.getDisjuncts(),
hasItems(new TermQuery(new Term(TEXT_FIELD_NAME, "hello")), new TermQuery(new Term(KEYWORD_FIELD_NAME, "hello")))
hasItems(new TermQuery(new Term(TEXT_FIELD_NAME, "hello")), new IndexOrDocValuesQuery(new TermQuery(new Term(KEYWORD_FIELD_NAME, "hello")), SortedSetDocValuesField.newSlowExactQuery(KEYWORD_FIELD_NAME, new BytesRef("hello"))))
);
}

Expand Down
Loading

0 comments on commit a7f6bd2

Please sign in to comment.