Skip to content

Commit

Permalink
[Derived Fields] PR4: Capability to define derived fields in search r…
Browse files Browse the repository at this point in the history 
…equest (opensearch-project#12850)

* Support derived fields definition in search request
* adds support for fetch phase on derived fields
* adds support for highlighting on derived fields

---------

Signed-off-by: Rishabh Maurya <[email protected]>
Signed-off-by: Shivansh Arora <[email protected]>
  • Loading branch information
@rishabhmaurya @shiv0408
rishabhmaurya authored and shiv0408 committed Apr 25, 2024
1 parent e01c4c7 commit 042a2d6
Show file tree
Hide file tree
Showing 26 changed files with 1,373 additions and 215 deletions.
229 changes: 229 additions & 0 deletions client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,15 +54,19 @@
import org.opensearch.action.search.SearchScrollRequest;
import org.opensearch.client.core.CountRequest;
import org.opensearch.client.core.CountResponse;
import org.opensearch.common.geo.ShapeRelation;
import org.opensearch.common.unit.TimeValue;
import org.opensearch.common.xcontent.XContentFactory;
import org.opensearch.core.common.bytes.BytesReference;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.core.xcontent.MediaTypeRegistry;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.geometry.Rectangle;
import org.opensearch.index.query.GeoShapeQueryBuilder;
import org.opensearch.index.query.MatchQueryBuilder;
import org.opensearch.index.query.QueryBuilder;
import org.opensearch.index.query.QueryBuilders;
import org.opensearch.index.query.RangeQueryBuilder;
import org.opensearch.index.query.ScriptQueryBuilder;
import org.opensearch.index.query.TermsQueryBuilder;
import org.opensearch.join.aggregations.Children;
Expand Down Expand Up @@ -102,6 +106,8 @@
import org.opensearch.search.suggest.Suggest;
import org.opensearch.search.suggest.SuggestBuilder;
import org.opensearch.search.suggest.phrase.PhraseSuggestionBuilder;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.hamcrest.Matchers;
import org.junit.Before;

Expand All @@ -116,6 +122,7 @@
import java.util.concurrent.TimeUnit;

import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.opensearch.index.query.QueryBuilders.geoShapeQuery;
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent;
import static org.hamcrest.Matchers.arrayContaining;
import static org.hamcrest.Matchers.both;
Expand Down Expand Up @@ -764,6 +771,228 @@ public void testSearchWithWeirdScriptFields() throws Exception {
}
}

public void testSearchWithDerivedFields() throws Exception {
// Just testing DerivedField definition from SearchSourceBuilder derivedField()
// We are not testing the full functionality here
Request doc = new Request("PUT", "test/_doc/1");
doc.setJsonEntity("{\"field\":\"value\"}");
client().performRequest(doc);
client().performRequest(new Request("POST", "/test/_refresh"));
// Keyword field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "keyword", new Script("emit(params._source[\"field\"])"))
.fetchField("result")
.query(new TermsQueryBuilder("result", "value"))
);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals("value", values.get(0));

// multi valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField(
"result",
"keyword",
new Script("emit(params._source[\"field\"]);emit(params._source[\"field\"] + \"_2\")")
)
.query(new TermsQueryBuilder("result", "value_2"))
.fetchField("result")
);
searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals("value", values.get(0));
assertEquals("value_2", values.get(1));
}
// Boolean field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "boolean", new Script("emit(((String)params._source[\"field\"]).equals(\"value\"))"))
.query(new TermsQueryBuilder("result", "true"))
.fetchField("result")
);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(true, values.get(0));
}
// Long field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "long", new Script("emit(Long.MAX_VALUE)"))
.query(new RangeQueryBuilder("result").from(Long.MAX_VALUE - 1).to(Long.MAX_VALUE))
.fetchField("result")
);

SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(Long.MAX_VALUE, values.get(0));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "long", new Script("emit(Long.MAX_VALUE); emit(Long.MIN_VALUE);"))
.query(new RangeQueryBuilder("result").from(Long.MIN_VALUE).to(Long.MIN_VALUE + 1))
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals(Long.MAX_VALUE, values.get(0));
assertEquals(Long.MIN_VALUE, values.get(1));
}
// Double field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "double", new Script("emit(Double.MAX_VALUE)"))
.query(new RangeQueryBuilder("result").from(Double.MAX_VALUE - 1).to(Double.MAX_VALUE))
.fetchField("result")
);
SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(Double.MAX_VALUE, values.get(0));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "double", new Script("emit(Double.MAX_VALUE); emit(Double.MIN_VALUE);"))
.query(new RangeQueryBuilder("result").from(Double.MIN_VALUE).to(Double.MIN_VALUE + 1))
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals(Double.MAX_VALUE, values.get(0));
assertEquals(Double.MIN_VALUE, values.get(1));
}
// Date field
{
DateTime date1 = new DateTime(1990, 12, 29, 0, 0, DateTimeZone.UTC);
DateTime date2 = new DateTime(1990, 12, 30, 0, 0, DateTimeZone.UTC);
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "date", new Script("emit(" + date1.getMillis() + "L)"))
.query(new RangeQueryBuilder("result").from(date1.toString()).to(date2.toString()))
.fetchField("result")
);

SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(date1.toString(), values.get(0));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "date", new Script("emit(" + date1.getMillis() + "L); " + "emit(" + date2.getMillis() + "L)"))
.query(new RangeQueryBuilder("result").from(date1.toString()).to(date2.toString()))
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals(date1.toString(), values.get(0));
assertEquals(date2.toString(), values.get(1));
}
// Geo field
{
GeoShapeQueryBuilder qb = geoShapeQuery("result", new Rectangle(-35, 35, 35, -35));
qb.relation(ShapeRelation.INTERSECTS);
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "geo_point", new Script("emit(10.0, 20.0)"))
.query(qb)
.fetchField("result")
);

SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals(10.0, ((HashMap) values.get(0)).get("lat"));
assertEquals(20.0, ((HashMap) values.get(0)).get("lon"));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "geo_point", new Script("emit(10.0, 20.0); emit(20.0, 30.0);"))
.query(qb)
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals(10.0, ((HashMap) values.get(0)).get("lat"));
assertEquals(20.0, ((HashMap) values.get(0)).get("lon"));
assertEquals(20.0, ((HashMap) values.get(1)).get("lat"));
assertEquals(30.0, ((HashMap) values.get(1)).get("lon"));
}
// IP field
{
SearchRequest searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource().derivedField("result", "ip", new Script("emit(\"10.0.0.1\")")).fetchField("result")
);

SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
SearchHit searchHit = searchResponse.getHits().getAt(0);
List<Object> values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(1, values.size());
assertEquals("10.0.0.1", values.get(0));

// multi-valued
searchRequest = new SearchRequest("test").source(
SearchSourceBuilder.searchSource()
.derivedField("result", "ip", new Script("emit(\"10.0.0.1\"); emit(\"10.0.0.2\");"))
.fetchField("result")
);

searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
searchHit = searchResponse.getHits().getAt(0);
values = searchHit.getFields().get("result").getValues();
assertNotNull(values);
assertEquals(2, values.size());
assertEquals("10.0.0.1", values.get(0));
assertEquals("10.0.0.2", values.get(1));

}

}

public void testSearchScroll() throws Exception {
for (int i = 0; i < 100; i++) {
XContentBuilder builder = jsonBuilder().startObject().field("field", i).endObject();
Expand Down
Loading

0 comments on commit 042a2d6

Please sign in to comment.