From 27cb5cd1cbaa8d254f0db9b1c2d2686a9713c9c2 Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Fri, 17 Nov 2023 14:44:26 -0800 Subject: [PATCH] Add indexRandomForConcurrentSearch to tests: * CardinalityWithRequestBreakerIT.java * TopHitsIT.java * SearchFieldsIT.java * DecayFunctionScoreIT.java * FunctionScoreFieldValueIT.java * FunctionScoreIT.java * FunctionScorePluginIT.java * QueryRescorerIT.java Signed-off-by: Jay Deng --- .../CardinalityWithRequestBreakerIT.java | 2 +- .../search/aggregations/metrics/TopHitsIT.java | 2 -- .../search/fields/SearchFieldsIT.java | 17 ++++++++++++++--- .../functionscore/DecayFunctionScoreIT.java | 18 ++++++++++-------- .../FunctionScoreFieldValueIT.java | 8 ++++---- .../search/functionscore/FunctionScoreIT.java | 14 +++++++++----- .../functionscore/FunctionScorePluginIT.java | 2 +- .../search/functionscore/QueryRescorerIT.java | 12 +++++++----- 8 files changed, 46 insertions(+), 29 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java index 66bd565d783d0..6cbf278317b1b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/CardinalityWithRequestBreakerIT.java @@ -100,7 +100,7 @@ public void testRequestBreaker() throws Exception { ) .get(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); try { client().prepareSearch("test") .addAggregation( diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TopHitsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TopHitsIT.java index de0b2d0e08108..10e51079cf389 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TopHitsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/TopHitsIT.java @@ -612,7 +612,6 @@ public void testSortByBucket() throws Exception { } public void testFieldCollapsing() throws Exception { - indexRandomForConcurrentSearch("field-collapsing"); SearchResponse response = client().prepareSearch("field-collapsing") .setSearchType(SearchType.DFS_QUERY_THEN_FETCH) .setQuery(matchQuery("text", "term rare")) @@ -754,7 +753,6 @@ public void testEmptyIndex() throws Exception { } public void testTrackScores() throws Exception { - indexRandomForConcurrentSearch(3, "field-collapsing"); boolean[] trackScores = new boolean[] { true, false }; for (boolean trackScore : trackScores) { logger.info("Track score={}", trackScore); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java index 8adcf5f1bb47c..ed8fe74504f92 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java @@ -251,6 +251,7 @@ public void testStoredFields() throws Exception { .get(); client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field1").get(); assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L)); @@ -358,6 +359,7 @@ public void testScriptDocAndFields() throws Exception { ) .get(); client().admin().indices().refresh(refreshRequest()).actionGet(); + indexRandomForConcurrentSearch("test"); logger.info("running doc['num1'].value"); SearchResponse response = client().prepareSearch() @@ -458,6 +460,7 @@ public void testScriptWithUnsignedLong() throws Exception { ) .get(); client().admin().indices().refresh(refreshRequest()).actionGet(); + indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch() .setQuery(matchAllQuery()) @@ -547,6 +550,7 @@ public void testScriptFieldWithNanos() throws Exception { .setSource(jsonBuilder().startObject().field("date", "1970-01-01T00:00:00.000Z").endObject()), client().prepareIndex("test").setId("2").setSource(jsonBuilder().startObject().field("date", date).endObject()) ); + indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch() .setQuery(matchAllQuery()) @@ -632,7 +636,7 @@ public void testScriptFieldUsingSource() throws Exception { ) .get(); client().admin().indices().refresh(refreshRequest()).actionGet(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch() .setQuery(matchAllQuery()) @@ -675,7 +679,7 @@ public void testScriptFieldUsingSource() throws Exception { public void testScriptFieldsForNullReturn() throws Exception { client().prepareIndex("test").setId("1").setSource("foo", "bar").setRefreshPolicy("true").get(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch() .setQuery(matchAllQuery()) .addScriptField("test_script_1", new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "return null", Collections.emptyMap())) @@ -797,6 +801,7 @@ public void testStoredFieldsWithoutSource() throws Exception { .get(); client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) @@ -854,6 +859,7 @@ public void testSearchFieldsMetadata() throws Exception { .setSource(jsonBuilder().startObject().field("field1", "value").endObject()) .setRefreshPolicy(IMMEDIATE) .get(); + indexRandomForConcurrentSearch("my-index"); SearchResponse searchResponse = client().prepareSearch("my-index").addStoredField("field1").addStoredField("_routing").get(); @@ -868,6 +874,7 @@ public void testSearchFieldsNonLeafField() throws Exception { .setSource(jsonBuilder().startObject().startObject("field1").field("field2", "value1").endObject().endObject()) .setRefreshPolicy(IMMEDIATE) .get(); + indexRandomForConcurrentSearch("my-index"); assertFailures( client().prepareSearch("my-index").addStoredField("field1"), @@ -934,6 +941,7 @@ public void testGetFieldsComplexField() throws Exception { ); client().prepareIndex("my-index").setId("1").setRefreshPolicy(IMMEDIATE).setSource(source, MediaTypeRegistry.JSON).get(); + indexRandomForConcurrentSearch("my-index"); String field = "field1.field2.field3.field4"; @@ -1041,6 +1049,7 @@ public void testDocValueFields() throws Exception { .get(); client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); SearchRequestBuilder builder = client().prepareSearch() .setQuery(matchAllQuery()) @@ -1391,6 +1400,7 @@ public void testWildcardDocValueFieldsWithFieldAlias() throws Exception { index("test", MapperService.SINGLE_MAPPING_NAME, "1", "text_field", "foo", "date_field", formatter.print(date)); refresh("test"); + indexRandomForConcurrentSearch("test"); SearchRequestBuilder builder = client().prepareSearch() .setQuery(matchAllQuery()) @@ -1444,7 +1454,7 @@ public void testStoredFieldsWithFieldAlias() throws Exception { index("test", MapperService.SINGLE_MAPPING_NAME, "1", "field1", "value1", "field2", "value2"); refresh("test"); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) @@ -1487,6 +1497,7 @@ public void testWildcardStoredFieldsWithFieldAlias() throws Exception { index("test", MapperService.SINGLE_MAPPING_NAME, "1", "field1", "value1", "field2", "value2"); refresh("test"); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch().setQuery(matchAllQuery()).addStoredField("field*").get(); assertHitCount(searchResponse, 1L); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/DecayFunctionScoreIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/DecayFunctionScoreIT.java index 16bfa56b69f97..3a6624c2ad2e6 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/DecayFunctionScoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/DecayFunctionScoreIT.java @@ -398,6 +398,7 @@ public void testBoostModeSettingWorks() throws Exception { ) ); indexRandom(true, false, indexBuilders); // force no dummy docs + indexRandomForConcurrentSearch("test"); // Test Gauss List lonlat = new ArrayList<>(); @@ -482,7 +483,7 @@ public void testParseGeoPoint() throws Exception { constantScoreQuery(termQuery("test", "value")), ScoreFunctionBuilders.weightFactorFunction(randomIntBetween(1, 10)) ); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); GeoPoint point = new GeoPoint(20, 11); ActionFuture response = client().search( searchRequest().searchType(SearchType.QUERY_THEN_FETCH) @@ -536,7 +537,7 @@ public void testCombineModes() throws Exception { .setRefreshPolicy(IMMEDIATE) .setSource(jsonBuilder().startObject().field("test", "value value").field("num", 1.0).endObject()) .get(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); FunctionScoreQueryBuilder baseQuery = functionScoreQuery( constantScoreQuery(termQuery("test", "value")), ScoreFunctionBuilders.weightFactorFunction(2) @@ -656,7 +657,7 @@ public void testCombineModesExplain() throws Exception { constantScoreQuery(termQuery("test", "value")).queryName("query1"), ScoreFunctionBuilders.weightFactorFunction(2, "weight1") ); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); // decay score should return 0.5 for this function and baseQuery should return 2.0f as it's score ActionFuture response = client().search( searchRequest().searchType(SearchType.QUERY_THEN_FETCH) @@ -765,7 +766,7 @@ public void testParseDateMath() throws Exception { ).actionGet(); refresh(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); SearchResponse sr = client().search( searchRequest().source( searchSource().query(functionScoreQuery(termQuery("test", "value"), gaussDecayFunction("num1", "now", "2d"))) @@ -821,7 +822,7 @@ public void testValueMissingLin() throws Exception { ).actionGet(); refresh(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); ActionFuture response = client().search( searchRequest().searchType(SearchType.QUERY_THEN_FETCH) @@ -898,6 +899,7 @@ public void testDateWithoutOrigin() throws Exception { ).actionGet(); refresh(); + indexRandomForConcurrentSearch("test"); ActionFuture response = client().search( searchRequest().searchType(SearchType.QUERY_THEN_FETCH) @@ -979,7 +981,7 @@ public void testManyDocsLin() throws Exception { List lonlat = new ArrayList<>(); lonlat.add(100f); lonlat.add(110f); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); ActionFuture response = client().search( searchRequest().searchType(SearchType.QUERY_THEN_FETCH) .source( @@ -1113,7 +1115,7 @@ public void testNoQueryGiven() throws Exception { client().index(indexRequest("test").source(jsonBuilder().startObject().field("test", "value").field("num", 1.0).endObject())) .actionGet(); refresh(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); // so, we indexed a string field, but now we try to score a num field ActionFuture response = client().search( searchRequest().searchType(SearchType.QUERY_THEN_FETCH) @@ -1178,7 +1180,7 @@ public void testMultiFieldOptions() throws Exception { ); indexRandom(true, doc1, doc2); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); ActionFuture response = client().search(searchRequest().source(searchSource().query(baseQuery))); SearchResponse sr = response.actionGet(); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreFieldValueIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreFieldValueIT.java index c00696d9acdec..d53f55b98bd23 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreFieldValueIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreFieldValueIT.java @@ -80,7 +80,7 @@ protected Settings featureFlagSettings() { return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); } - public void testFieldValueFactor() throws IOException { + public void testFieldValueFactor() throws IOException, InterruptedException { assertAcked( prepareCreate("test").setMapping( jsonBuilder().startObject() @@ -99,8 +99,8 @@ public void testFieldValueFactor() throws IOException { client().prepareIndex("test").setId("1").setSource("test", 5, "body", "foo").get(); client().prepareIndex("test").setId("2").setSource("test", 17, "body", "foo").get(); client().prepareIndex("test").setId("3").setSource("body", "bar").get(); - refresh(); + indexRandomForConcurrentSearch("test"); // document 2 scores higher because 17 > 5 SearchResponse response = client().prepareSearch("test") @@ -189,7 +189,7 @@ public void testFieldValueFactor() throws IOException { } } - public void testFieldValueFactorExplain() throws IOException { + public void testFieldValueFactorExplain() throws IOException, InterruptedException { assertAcked( prepareCreate("test").setMapping( jsonBuilder().startObject() @@ -210,7 +210,7 @@ public void testFieldValueFactorExplain() throws IOException { client().prepareIndex("test").setId("3").setSource("body", "bar").get(); refresh(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); // document 2 scores higher because 17 > 5 final String functionName = "func1"; diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreIT.java index 37b1775cbc378..3b80d437e95c0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScoreIT.java @@ -126,11 +126,11 @@ protected Map, Object>> pluginScripts() { } } - public void testScriptScoresNested() throws IOException { + public void testScriptScoresNested() throws IOException, InterruptedException { createIndex(INDEX); index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); refresh(); - indexRandomForConcurrentSearch(3, INDEX); + indexRandomForConcurrentSearch(INDEX); Script scriptOne = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "1", Collections.emptyMap()); Script scriptTwo = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap()); @@ -149,10 +149,11 @@ public void testScriptScoresNested() throws IOException { assertThat(response.getHits().getAt(0).getScore(), equalTo(1.0f)); } - public void testScriptScoresWithAgg() throws IOException { + public void testScriptScoresWithAgg() throws IOException, InterruptedException { createIndex(INDEX); index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); refresh(); + indexRandomForConcurrentSearch(INDEX); Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap()); @@ -167,10 +168,11 @@ public void testScriptScoresWithAgg() throws IOException { assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), is(1L)); } - public void testScriptScoresWithAggWithExplain() throws IOException { + public void testScriptScoresWithAggWithExplain() throws IOException, InterruptedException { createIndex(INDEX); index(INDEX, TYPE, "1", jsonBuilder().startObject().field("dummy_field", 1).endObject()); refresh(); + indexRandomForConcurrentSearch(INDEX); Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "get score value", Collections.emptyMap()); @@ -196,7 +198,7 @@ public void testScriptScoresWithAggWithExplain() throws IOException { assertThat(((Terms) response.getAggregations().asMap().get("score_agg")).getBuckets().get(0).getDocCount(), is(1L)); } - public void testMinScoreFunctionScoreBasic() throws IOException { + public void testMinScoreFunctionScoreBasic() throws IOException, InterruptedException { float score = randomValueOtherThanMany((f) -> Float.compare(f, 0) < 0, OpenSearchTestCase::randomFloat); float minScore = randomValueOtherThanMany((f) -> Float.compare(f, 0) < 0, OpenSearchTestCase::randomFloat); index( @@ -208,6 +210,7 @@ public void testMinScoreFunctionScoreBasic() throws IOException { .endObject() ); refresh(); + indexRandomForConcurrentSearch(INDEX); ensureYellow(); Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['random_score']", Collections.emptyMap()); @@ -292,6 +295,7 @@ public void testWithEmptyFunctions() throws IOException, ExecutionException, Int assertAcked(prepareCreate("test")); index("test", "testtype", "1", jsonBuilder().startObject().field("text", "test text").endObject()); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse termQuery = client().search(searchRequest().source(searchSource().explain(true).query(termQuery("text", "text")))) .get(); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java index 29026607a5687..a91f53dae04d2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/FunctionScorePluginIT.java @@ -122,7 +122,7 @@ public void testPlugin() throws Exception { ).actionGet(); client().admin().indices().prepareRefresh().get(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); DecayFunctionBuilder gfb = new CustomDistanceScoreBuilder("num1", "2013-05-28", "+1d"); ActionFuture response = client().search( diff --git a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/QueryRescorerIT.java b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/QueryRescorerIT.java index 0a488566135a9..bda6284d9535a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/functionscore/QueryRescorerIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/functionscore/QueryRescorerIT.java @@ -110,7 +110,7 @@ protected Settings featureFlagSettings() { return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build(); } - public void testEnforceWindowSize() { + public void testEnforceWindowSize() throws InterruptedException { createIndex("test"); // this int iters = scaledRandomIntBetween(10, 20); @@ -118,7 +118,7 @@ public void testEnforceWindowSize() { client().prepareIndex("test").setId(Integer.toString(i)).setSource("f", Integer.toString(i)).get(); } refresh(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); int numShards = getNumShards("test").numPrimaries; for (int j = 0; j < iters; j++) { @@ -170,7 +170,7 @@ public void testRescorePhrase() throws Exception { .setSource("field1", "quick huge brown", "field2", "the quick lazy huge brown fox jumps over the tree") .get(); refresh(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setQuery(QueryBuilders.matchQuery("field1", "the quick brown").operator(Operator.OR)) .setRescorer( @@ -476,6 +476,7 @@ private static void assertEquivalent(String query, SearchResponse plain, SearchR public void testEquivalence() throws Exception { // no dummy docs since merges can change scores while we run queries. int numDocs = indexRandomNumbers("whitespace", -1, false); + indexRandomForConcurrentSearch("test"); final int iters = scaledRandomIntBetween(50, 100); for (int i = 0; i < iters; i++) { @@ -547,7 +548,7 @@ public void testExplain() throws Exception { .setSource("field1", "quick huge brown", "field2", "the quick lazy huge brown fox jumps over the tree") .get(); refresh(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); { SearchResponse searchResponse = client().prepareSearch() @@ -819,7 +820,7 @@ public void testFromSize() throws Exception { client().prepareIndex("test").setId("" + i).setSource("text", "hello world").get(); } refresh(); - indexRandomForConcurrentSearch(3, "test"); + indexRandomForConcurrentSearch("test"); SearchRequestBuilder request = client().prepareSearch(); request.setQuery(QueryBuilders.termQuery("text", "hello")); @@ -836,6 +837,7 @@ public void testRescorePhaseWithInvalidSort() throws Exception { client().prepareIndex("test").setId("" + i).setSource("number", 0).get(); } refresh(); + indexRandomForConcurrentSearch("test"); Exception exc = expectThrows( Exception.class,