Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Derived Fields] PR4: Capability to define derived fields in search request #12850

Merged
merged 14 commits into from
Apr 11, 2024
Merged

[Derived Fields] PR4: Capability to define derived fields in search request #12850

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
cff0b06
Support derived fields definition in search request
rishabhmaurya Mar 22, 2024
e895fc8
Add tests related to derived fields for QueryShardContext changes
rishabhmaurya Apr 3, 2024
8acbba6
Add tests and refactor code
rishabhmaurya Apr 4, 2024
31377f6
Refactor code
rishabhmaurya Apr 4, 2024
9dec815
Fix test
rishabhmaurya Apr 4, 2024
a08aa1a
Fix backward compatibility
rishabhmaurya Apr 4, 2024
b2662e7
Add tests for fetch phase
rishabhmaurya Apr 5, 2024
b911d49
Fix current version to fix backward comp test
rishabhmaurya Apr 5, 2024
72c1d9d
Fix failing test
rishabhmaurya Apr 5, 2024
942832c
Add test for fetch and highlight phase for derived fields
rishabhmaurya Apr 6, 2024
57299ea
Integ and unit tests
rishabhmaurya Apr 10, 2024
26e361f
remove unused code
rishabhmaurya Apr 10, 2024
0cc00ea
Fix the geo_point unit test
rishabhmaurya Apr 10, 2024
dd4ca2a
Add unit test for serialization of derived fields from query
rishabhmaurya Apr 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
229 changes: 229 additions & 0 deletions client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,19 @@
import org.opensearch.action.search.SearchScrollRequest;
import org.opensearch.client.core.CountRequest;
import org.opensearch.client.core.CountResponse;
import org.opensearch.common.geo.ShapeRelation;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.geometry.Rectangle;
import org.opensearch.index.query.GeoShapeQueryBuilder;
import org.opensearch.index.query.MatchQueryBuilder;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.index.query.RangeQueryBuilder;
import org.opensearch.index.query.ScriptQueryBuilder;
import org.opensearch.index.query.TermsQueryBuilder;
import org.opensearch.join.aggregations.Children;
Expand Down Expand Up @@ -102,6 +106,8 @@
import org.opensearch.search.suggest.Suggest;
import org.opensearch.search.suggest.SuggestBuilder;
import org.opensearch.search.suggest.phrase.PhraseSuggestionBuilder;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.hamcrest.Matchers;
import org.junit.Before;

Expand All @@ -116,6 +122,7 @@
import java.util.concurrent.TimeUnit;

import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.opensearch.index.query.QueryBuilders.geoShapeQuery;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent;
import static org.hamcrest.Matchers.arrayContaining;
import static org.hamcrest.Matchers.both;
Expand Down Expand Up @@ -764,6 +771,228 @@ public void testSearchWithWeirdScriptFields() throws Exception {
}
}

public void testSearchWithDerivedFields() throws Exception {
// Just testing DerivedField definition from SearchSourceBuilder derivedField()
// We are not testing the full functionality here
Request doc = new Request("PUT", "test/_doc/1");
doc.setJsonEntity("{\"field\":\"value\"}");
client().performRequest(doc);
client().performRequest(new Request("POST", "/test/_refresh"));
// Keyword field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "keyword", new Script("emit(params._source[\"field\"])"))
.fetchField("result")
.query(new TermsQueryBuilder("result", "value"))
);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals("value", values.get(0));

// multi valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField(
"result",
"keyword",
new Script("emit(params._source[\"field\"]);emit(params._source[\"field\"] + \"_2\")")
)
.query(new TermsQueryBuilder("result", "value_2"))
.fetchField("result")
);
searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals("value", values.get(0));
assertEquals("value_2", values.get(1));
}
// Boolean field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "boolean", new Script("emit(((String)params._source[\"field\"]).equals(\"value\"))"))
.query(new TermsQueryBuilder("result", "true"))
.fetchField("result")
);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(true, values.get(0));
}
// Long field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "long", new Script("emit(Long.MAX_VALUE)"))
.query(new RangeQueryBuilder("result").from(Long.MAX_VALUE - 1).to(Long.MAX_VALUE))
.fetchField("result")
);

SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(Long.MAX_VALUE, values.get(0));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "long", new Script("emit(Long.MAX_VALUE); emit(Long.MIN_VALUE);"))
.query(new RangeQueryBuilder("result").from(Long.MIN_VALUE).to(Long.MIN_VALUE + 1))
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals(Long.MAX_VALUE, values.get(0));
assertEquals(Long.MIN_VALUE, values.get(1));
}
// Double field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "double", new Script("emit(Double.MAX_VALUE)"))
.query(new RangeQueryBuilder("result").from(Double.MAX_VALUE - 1).to(Double.MAX_VALUE))
.fetchField("result")
);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(Double.MAX_VALUE, values.get(0));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "double", new Script("emit(Double.MAX_VALUE); emit(Double.MIN_VALUE);"))
.query(new RangeQueryBuilder("result").from(Double.MIN_VALUE).to(Double.MIN_VALUE + 1))
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals(Double.MAX_VALUE, values.get(0));
assertEquals(Double.MIN_VALUE, values.get(1));
}
// Date field
{
DateTime date1 = new DateTime(1990, 12, 29, 0, 0, DateTimeZone.UTC);
DateTime date2 = new DateTime(1990, 12, 30, 0, 0, DateTimeZone.UTC);
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "date", new Script("emit(" + date1.getMillis() + "L)"))
.query(new RangeQueryBuilder("result").from(date1.toString()).to(date2.toString()))
.fetchField("result")
);

SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(date1.toString(), values.get(0));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "date", new Script("emit(" + date1.getMillis() + "L); " + "emit(" + date2.getMillis() + "L)"))
.query(new RangeQueryBuilder("result").from(date1.toString()).to(date2.toString()))
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals(date1.toString(), values.get(0));
assertEquals(date2.toString(), values.get(1));
}
// Geo field
{
GeoShapeQueryBuilder qb = geoShapeQuery("result", new Rectangle(-35, 35, 35, -35));
qb.relation(ShapeRelation.INTERSECTS);
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "geo_point", new Script("emit(10.0, 20.0)"))
.query(qb)
.fetchField("result")
);

SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(10.0, ((HashMap) values.get(0)).get("lat"));
assertEquals(20.0, ((HashMap) values.get(0)).get("lon"));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "geo_point", new Script("emit(10.0, 20.0); emit(20.0, 30.0);"))
.query(qb)
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals(10.0, ((HashMap) values.get(0)).get("lat"));
assertEquals(20.0, ((HashMap) values.get(0)).get("lon"));
assertEquals(20.0, ((HashMap) values.get(1)).get("lat"));
assertEquals(30.0, ((HashMap) values.get(1)).get("lon"));
}
// IP field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource().derivedField("result", "ip", new Script("emit(\"10.0.0.1\")")).fetchField("result")
);

SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals("10.0.0.1", values.get(0));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "ip", new Script("emit(\"10.0.0.1\"); emit(\"10.0.0.2\");"))
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals("10.0.0.1", values.get(0));
assertEquals("10.0.0.2", values.get(1));

}

}

public void testSearchScroll() throws Exception {
for (int i = 0; i < 100; i++) {
XContentBuilder builder = jsonBuilder().startObject().field("field", i).endObject();
Expand Down
Loading
Loading