From a120dd283cd174327494ef0cc63f377dd029ab7f Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Tue, 9 Aug 2022 09:58:52 -0700 Subject: [PATCH 01/27] Add IndexShard#getLatestReplicationCheckpoint behind segrep enable feature flag (#4163) * Add IndexShard#getLatestReplicationCheckpoint behind segrep enable feature flag Signed-off-by: Suraj Singh * Address review comment. Move tests to SegmentReplicationIndexShardTests Signed-off-by: Suraj Singh * Add segrep enbaled index settings in TargetServiceTests, SourceHandlerTests Signed-off-by: Suraj Singh --- .../opensearch/index/shard/IndexShard.java | 7 ++++-- .../SegmentReplicationIndexShardTests.java | 24 +++++++++++++++++++ .../SegmentReplicationSourceHandlerTests.java | 5 +++- .../SegmentReplicationTargetServiceTests.java | 6 ++++- 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index b05eec4304cd5..53324c94c4b08 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -1396,10 +1396,13 @@ public GatedCloseable acquireSafeIndexCommit() throws EngineExcepti } /** - * Returns the lastest Replication Checkpoint that shard received. Shards will return an EMPTY checkpoint before - * the engine is opened. + * Returns the latest ReplicationCheckpoint that shard received. + * @return EMPTY checkpoint before the engine is opened and null for non-segrep enabled indices */ public ReplicationCheckpoint getLatestReplicationCheckpoint() { + if (indexSettings.isSegRepEnabled() == false) { + return null; + } if (getEngineOrNull() == null) { return ReplicationCheckpoint.empty(shardId); } diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 3fcf6116b11a2..4f2784db93df2 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -14,14 +14,38 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationType; +import java.io.IOException; + public class SegmentReplicationIndexShardTests extends OpenSearchIndexLevelReplicationTestCase { private static final Settings settings = Settings.builder() .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) .build(); + /** + * Test that latestReplicationCheckpoint returns null only for docrep enabled indices + */ + public void testReplicationCheckpointNullForDocRep() throws IOException { + Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, "DOCUMENT").put(Settings.EMPTY).build(); + final IndexShard indexShard = newStartedShard(false, indexSettings); + assertNull(indexShard.getLatestReplicationCheckpoint()); + closeShards(indexShard); + } + + /** + * Test that latestReplicationCheckpoint returns ReplicationCheckpoint for segrep enabled indices + */ + public void testReplicationCheckpointNotNullForSegReb() throws IOException { + Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, "SEGMENT").put(Settings.EMPTY).build(); + final IndexShard indexShard = newStartedShard(indexSettings); + final ReplicationCheckpoint replicationCheckpoint = indexShard.getLatestReplicationCheckpoint(); + assertNotNull(replicationCheckpoint); + closeShards(indexShard); + } + public void testIgnoreShardIdle() throws Exception { try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) { shards.startAll(); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java index 70061c54d0da2..535cd5974490f 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java @@ -15,7 +15,9 @@ import org.opensearch.OpenSearchException; import org.opensearch.Version; import org.opensearch.action.ActionListener; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.common.settings.Settings; import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.IndexShardTestCase; import org.opensearch.index.store.StoreFileMetadata; @@ -41,7 +43,8 @@ public class SegmentReplicationSourceHandlerTests extends IndexShardTestCase { @Override public void setUp() throws Exception { super.setUp(); - primary = newStartedShard(true); + final Settings settings = Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, "SEGMENT").put(Settings.EMPTY).build(); + primary = newStartedShard(true, settings); replica = newShard(primary.shardId(), false); recoverReplica(replica, primary, true); replicaDiscoveryNode = replica.recoveryState().getTargetNode(); diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index 8b4bda7de50ad..b7cfda5dc1d64 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -13,6 +13,7 @@ import org.mockito.Mockito; import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.index.shard.IndexShard; @@ -50,7 +51,10 @@ public class SegmentReplicationTargetServiceTests extends IndexShardTestCase { @Override public void setUp() throws Exception { super.setUp(); - final Settings settings = Settings.builder().put("node.name", SegmentReplicationTargetServiceTests.class.getSimpleName()).build(); + final Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_REPLICATION_TYPE, "SEGMENT") + .put("node.name", SegmentReplicationTargetServiceTests.class.getSimpleName()) + .build(); final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings); final TransportService transportService = mock(TransportService.class); From 4751f3bd296e6952b852a457a82220ec248b037a Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Tue, 9 Aug 2022 12:56:14 -0700 Subject: [PATCH 02/27] Refactor the src and test of GeoBoundsAggregation on GeoPoint from server folder to geo module. Refactors the GeoBoundsAggregation for geo_point types from the core server to the geo module. Signed-off-by: Navneet Verma --- .../client/RestHighLevelClient.java | 3 - modules/geo/build.gradle | 4 +- .../geo/GeoModulePluginIntegTestCase.java | 47 +++ .../opensearch/geo/search/MissingValueIT.java | 59 ++++ ...ractGeoAggregatorTestCaseModulePlugin.java | 295 ++++++++++++++++++ .../aggregations/metrics/GeoBoundsIT.java | 23 +- .../org/opensearch/geo/GeoModulePlugin.java | 20 +- .../metrics/AbstractGeoBoundsAggregator.java | 128 ++++++++ .../aggregations/metrics/GeoBounds.java | 2 +- .../metrics/GeoBoundsAggregationBuilder.java | 16 +- .../metrics/GeoBoundsAggregator.java | 92 +----- .../metrics/GeoBoundsAggregatorFactory.java | 2 +- .../metrics/InternalGeoBounds.java | 2 +- .../aggregations/metrics/ParsedGeoBounds.java | 2 +- .../GeoBoundsAggregationBuilderTests.java | 43 +++ .../metrics/GeoBoundsAggregatorTests.java | 26 +- .../metrics/InternalGeoBoundsTests.java | 33 +- .../geo/tests/common/AggregationBuilders.java | 21 ++ .../common/AggregationInspectionHelper.java | 18 ++ .../geo/tests/common/RandomGeoGenerator.java | 86 +++++ .../search/aggregations/MissingValueIT.java | 24 -- .../org/opensearch/search/SearchModule.java | 8 - .../aggregations/AggregationBuilders.java | 9 - .../support/AggregationInspectionHelper.java | 5 - .../aggregations/AggregationsTests.java | 2 - .../aggregations/metrics/GeoBoundsTests.java | 53 ---- .../test/InternalAggregationTestCase.java | 3 - .../test/OpenSearchIntegTestCase.java | 1 + 28 files changed, 799 insertions(+), 228 deletions(-) create mode 100644 modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java create mode 100644 modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java create mode 100644 modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorTestCaseModulePlugin.java rename {server/src/internalClusterTest/java/org/opensearch => modules/geo/src/internalClusterTest/java/org/opensearch/geo}/search/aggregations/metrics/GeoBoundsIT.java (97%) create mode 100644 modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoBoundsAggregator.java rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/metrics/GeoBounds.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/metrics/GeoBoundsAggregationBuilder.java (93%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/metrics/GeoBoundsAggregator.java (51%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/metrics/GeoBoundsAggregatorFactory.java (98%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/metrics/InternalGeoBounds.java (99%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/metrics/ParsedGeoBounds.java (98%) create mode 100644 modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilderTests.java rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/metrics/GeoBoundsAggregatorTests.java (88%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/metrics/InternalGeoBoundsTests.java (81%) create mode 100644 modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java create mode 100644 modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java create mode 100644 modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java delete mode 100644 server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsTests.java diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java index d293b979debb5..7ae8f8826c5a4 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java @@ -157,7 +157,6 @@ import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder; import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; -import org.opensearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.opensearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; import org.opensearch.search.aggregations.metrics.InternalHDRPercentileRanks; import org.opensearch.search.aggregations.metrics.InternalHDRPercentiles; @@ -169,7 +168,6 @@ import org.opensearch.search.aggregations.metrics.ParsedAvg; import org.opensearch.search.aggregations.metrics.ParsedCardinality; import org.opensearch.search.aggregations.metrics.ParsedExtendedStats; -import org.opensearch.search.aggregations.metrics.ParsedGeoBounds; import org.opensearch.search.aggregations.metrics.ParsedGeoCentroid; import org.opensearch.search.aggregations.metrics.ParsedHDRPercentileRanks; import org.opensearch.search.aggregations.metrics.ParsedHDRPercentiles; @@ -2116,7 +2114,6 @@ static List getDefaultNamedXContents() { map.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c)); map.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c)); map.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c)); - map.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c)); map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c)); map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c)); map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c)); diff --git a/modules/geo/build.gradle b/modules/geo/build.gradle index 1146b6924b2f8..0b8e623c24ac6 100644 --- a/modules/geo/build.gradle +++ b/modules/geo/build.gradle @@ -28,9 +28,10 @@ * under the License. */ apply plugin: 'opensearch.yaml-rest-test' +apply plugin: 'opensearch.internal-cluster-test' opensearchplugin { - description 'Placeholder plugin for geospatial features in OpenSearch. only registers geo_shape field mapper for now' + description 'Plugin for geospatial features in OpenSearch. Registering the geo_shape and aggregations GeoBounds on Geo_Shape and Geo_Point' classname 'org.opensearch.geo.GeoModulePlugin' } @@ -42,4 +43,3 @@ restResources { artifacts { restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test')) } -test.enabled = false diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java new file mode 100644 index 0000000000000..7dc6f2c1b89b7 --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java @@ -0,0 +1,47 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo; + +import org.opensearch.index.mapper.GeoShapeFieldMapper; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.OpenSearchIntegTestCase; +import org.opensearch.test.TestGeoShapeFieldMapperPlugin; + +import java.util.Collection; +import java.util.Collections; + +/** + * This is the base class for all the Geo related integration tests. Use this class to add the features and settings + * for the test cluster on which integration tests are running. + */ +public abstract class GeoModulePluginIntegTestCase extends OpenSearchIntegTestCase { + /** + * Returns a collection of plugins that should be loaded on each node for doing the integration tests. As this + * geo plugin is not getting packaged in a zip, we need to load it before the tests run. + * + * @return List of {@link Plugin} + */ + @Override + protected Collection> nodePlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } + + /** + * This was added as a backdoor to Mock the implementation of {@link GeoShapeFieldMapper} which was coming from + * {@link GeoModulePlugin}. Mock implementation is {@link TestGeoShapeFieldMapperPlugin}. Now we are using the + * {@link GeoModulePlugin} in our integration tests we need to override this functionality to avoid multiple mapper + * error. + * + * @return boolean + */ + @Override + protected boolean addMockGeoShapeFieldMapper() { + return false; + } +} diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java new file mode 100644 index 0000000000000..2ac73728b2dab --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/MissingValueIT.java @@ -0,0 +1,59 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.search.aggregations.metrics.GeoBounds; +import org.opensearch.geo.tests.common.AggregationBuilders; +import org.opensearch.test.OpenSearchIntegTestCase; + +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.closeTo; + +@OpenSearchIntegTestCase.SuiteScopeTestCase +public class MissingValueIT extends GeoModulePluginIntegTestCase { + + @Override + protected void setupSuiteScopeCluster() throws Exception { + assertAcked(prepareCreate("idx").setMapping("date", "type=date", "location", "type=geo_point", "str", "type=keyword").get()); + indexRandom( + true, + client().prepareIndex("idx").setId("1").setSource(), + client().prepareIndex("idx") + .setId("2") + .setSource("str", "foo", "long", 3L, "double", 5.5, "date", "2015-05-07", "location", "1,2") + ); + } + + public void testUnmappedGeoBounds() { + SearchResponse response = client().prepareSearch("idx") + .addAggregation(AggregationBuilders.geoBounds("bounds").field("non_existing_field").missing("2,1")) + .get(); + assertSearchResponse(response); + GeoBounds bounds = response.getAggregations().get("bounds"); + assertThat(bounds.bottomRight().lat(), closeTo(2.0, 1E-5)); + assertThat(bounds.bottomRight().lon(), closeTo(1.0, 1E-5)); + assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); + assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); + } + + public void testGeoBounds() { + SearchResponse response = client().prepareSearch("idx") + .addAggregation(AggregationBuilders.geoBounds("bounds").field("location").missing("2,1")) + .get(); + assertSearchResponse(response); + GeoBounds bounds = response.getAggregations().get("bounds"); + assertThat(bounds.bottomRight().lat(), closeTo(1.0, 1E-5)); + assertThat(bounds.bottomRight().lon(), closeTo(2.0, 1E-5)); + assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); + assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); + } +} diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorTestCaseModulePlugin.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorTestCaseModulePlugin.java new file mode 100644 index 0000000000000..0065cca7d6101 --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorTestCaseModulePlugin.java @@ -0,0 +1,295 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.metrics; + +import com.carrotsearch.hppc.ObjectIntHashMap; +import com.carrotsearch.hppc.ObjectIntMap; +import com.carrotsearch.hppc.ObjectObjectHashMap; +import com.carrotsearch.hppc.ObjectObjectMap; +import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.Strings; +import org.opensearch.common.document.DocumentField; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.ToXContent; +import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.geometry.utils.Geohash; +import org.opensearch.search.SearchHit; +import org.opensearch.search.sort.SortBuilders; +import org.opensearch.search.sort.SortOrder; + +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; + +/** + * This is base class for all Geo Aggregations Integration Tests. This class is similar to what we have in the server + * folder of the OpenSearch repo. As part of moving the Geo based aggregation into separate module and plugin we need + * to copy the code as we cannot depend on this class. + * GitHub issue + */ +public abstract class AbstractGeoAggregatorTestCaseModulePlugin extends GeoModulePluginIntegTestCase { + + protected static final String SINGLE_VALUED_FIELD_NAME = "geo_value"; + protected static final String MULTI_VALUED_FIELD_NAME = "geo_values"; + protected static final String NUMBER_FIELD_NAME = "l_values"; + protected static final String UNMAPPED_IDX_NAME = "idx_unmapped"; + protected static final String IDX_NAME = "idx"; + protected static final String EMPTY_IDX_NAME = "empty_idx"; + protected static final String DATELINE_IDX_NAME = "dateline_idx"; + protected static final String HIGH_CARD_IDX_NAME = "high_card_idx"; + protected static final String IDX_ZERO_NAME = "idx_zero"; + + protected static int numDocs; + protected static int numUniqueGeoPoints; + protected static GeoPoint[] singleValues, multiValues; + protected static GeoPoint singleTopLeft, singleBottomRight, multiTopLeft, multiBottomRight, singleCentroid, multiCentroid, + unmappedCentroid; + protected static ObjectIntMap expectedDocCountsForGeoHash = null; + protected static ObjectObjectMap expectedCentroidsForGeoHash = null; + protected static final double GEOHASH_TOLERANCE = 1E-5D; + + @Override + public void setupSuiteScopeCluster() throws Exception { + createIndex(UNMAPPED_IDX_NAME); + assertAcked( + prepareCreate(IDX_NAME).setMapping( + SINGLE_VALUED_FIELD_NAME, + "type=geo_point", + MULTI_VALUED_FIELD_NAME, + "type=geo_point", + NUMBER_FIELD_NAME, + "type=long", + "tag", + "type=keyword" + ) + ); + + singleTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); + singleBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); + multiTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); + multiBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); + singleCentroid = new GeoPoint(0, 0); + multiCentroid = new GeoPoint(0, 0); + unmappedCentroid = new GeoPoint(0, 0); + + numDocs = randomIntBetween(6, 20); + numUniqueGeoPoints = randomIntBetween(1, numDocs); + expectedDocCountsForGeoHash = new ObjectIntHashMap<>(numDocs * 2); + expectedCentroidsForGeoHash = new ObjectObjectHashMap<>(numDocs * 2); + + singleValues = new GeoPoint[numUniqueGeoPoints]; + for (int i = 0; i < singleValues.length; i++) { + singleValues[i] = RandomGeoGenerator.randomPoint(random()); + updateBoundsTopLeft(singleValues[i], singleTopLeft); + updateBoundsBottomRight(singleValues[i], singleBottomRight); + } + + multiValues = new GeoPoint[numUniqueGeoPoints]; + for (int i = 0; i < multiValues.length; i++) { + multiValues[i] = RandomGeoGenerator.randomPoint(random()); + updateBoundsTopLeft(multiValues[i], multiTopLeft); + updateBoundsBottomRight(multiValues[i], multiBottomRight); + } + + List builders = new ArrayList<>(); + + GeoPoint singleVal; + final GeoPoint[] multiVal = new GeoPoint[2]; + double newMVLat, newMVLon; + for (int i = 0; i < numDocs; i++) { + singleVal = singleValues[i % numUniqueGeoPoints]; + multiVal[0] = multiValues[i % numUniqueGeoPoints]; + multiVal[1] = multiValues[(i + 1) % numUniqueGeoPoints]; + builders.add( + client().prepareIndex(IDX_NAME) + .setSource( + jsonBuilder().startObject() + .array(SINGLE_VALUED_FIELD_NAME, singleVal.lon(), singleVal.lat()) + .startArray(MULTI_VALUED_FIELD_NAME) + .startArray() + .value(multiVal[0].lon()) + .value(multiVal[0].lat()) + .endArray() + .startArray() + .value(multiVal[1].lon()) + .value(multiVal[1].lat()) + .endArray() + .endArray() + .field(NUMBER_FIELD_NAME, i) + .field("tag", "tag" + i) + .endObject() + ) + ); + singleCentroid = singleCentroid.reset( + singleCentroid.lat() + (singleVal.lat() - singleCentroid.lat()) / (i + 1), + singleCentroid.lon() + (singleVal.lon() - singleCentroid.lon()) / (i + 1) + ); + newMVLat = (multiVal[0].lat() + multiVal[1].lat()) / 2d; + newMVLon = (multiVal[0].lon() + multiVal[1].lon()) / 2d; + multiCentroid = multiCentroid.reset( + multiCentroid.lat() + (newMVLat - multiCentroid.lat()) / (i + 1), + multiCentroid.lon() + (newMVLon - multiCentroid.lon()) / (i + 1) + ); + } + + assertAcked(prepareCreate(EMPTY_IDX_NAME).setMapping(SINGLE_VALUED_FIELD_NAME, "type=geo_point")); + + assertAcked( + prepareCreate(DATELINE_IDX_NAME).setMapping( + SINGLE_VALUED_FIELD_NAME, + "type=geo_point", + MULTI_VALUED_FIELD_NAME, + "type=geo_point", + NUMBER_FIELD_NAME, + "type=long", + "tag", + "type=keyword" + ) + ); + + GeoPoint[] geoValues = new GeoPoint[5]; + geoValues[0] = new GeoPoint(38, 178); + geoValues[1] = new GeoPoint(12, -179); + geoValues[2] = new GeoPoint(-24, 170); + geoValues[3] = new GeoPoint(32, -175); + geoValues[4] = new GeoPoint(-11, 178); + + for (int i = 0; i < 5; i++) { + builders.add( + client().prepareIndex(DATELINE_IDX_NAME) + .setSource( + jsonBuilder().startObject() + .array(SINGLE_VALUED_FIELD_NAME, geoValues[i].lon(), geoValues[i].lat()) + .field(NUMBER_FIELD_NAME, i) + .field("tag", "tag" + i) + .endObject() + ) + ); + } + assertAcked( + prepareCreate(HIGH_CARD_IDX_NAME).setSettings(Settings.builder().put("number_of_shards", 2)) + .setMapping( + SINGLE_VALUED_FIELD_NAME, + "type=geo_point", + MULTI_VALUED_FIELD_NAME, + "type=geo_point", + NUMBER_FIELD_NAME, + "type=long,store=true", + "tag", + "type=keyword" + ) + ); + + for (int i = 0; i < 2000; i++) { + singleVal = singleValues[i % numUniqueGeoPoints]; + builders.add( + client().prepareIndex(HIGH_CARD_IDX_NAME) + .setSource( + jsonBuilder().startObject() + .array(SINGLE_VALUED_FIELD_NAME, singleVal.lon(), singleVal.lat()) + .startArray(MULTI_VALUED_FIELD_NAME) + .startArray() + .value(multiValues[i % numUniqueGeoPoints].lon()) + .value(multiValues[i % numUniqueGeoPoints].lat()) + .endArray() + .startArray() + .value(multiValues[(i + 1) % numUniqueGeoPoints].lon()) + .value(multiValues[(i + 1) % numUniqueGeoPoints].lat()) + .endArray() + .endArray() + .field(NUMBER_FIELD_NAME, i) + .field("tag", "tag" + i) + .endObject() + ) + ); + updateGeohashBucketsCentroid(singleVal); + } + + builders.add( + client().prepareIndex(IDX_ZERO_NAME) + .setSource(jsonBuilder().startObject().array(SINGLE_VALUED_FIELD_NAME, 0.0, 1.0).endObject()) + ); + assertAcked(prepareCreate(IDX_ZERO_NAME).setMapping(SINGLE_VALUED_FIELD_NAME, "type=geo_point")); + + indexRandom(true, builders); + ensureSearchable(); + + // Added to debug a test failure where the terms aggregation seems to be reporting two documents with the same + // value for NUMBER_FIELD_NAME. This will check that after random indexing each document only has 1 value for + // NUMBER_FIELD_NAME and it is the correct value. Following this initial change its seems that this call was getting + // more that 2000 hits (actual value was 2059) so now it will also check to ensure all hits have the correct index and type. + SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) + .addStoredField(NUMBER_FIELD_NAME) + .addSort(SortBuilders.fieldSort(NUMBER_FIELD_NAME).order(SortOrder.ASC)) + .setSize(5000) + .get(); + assertSearchResponse(response); + long totalHits = response.getHits().getTotalHits().value; + XContentBuilder builder = XContentFactory.jsonBuilder(); + response.toXContent(builder, ToXContent.EMPTY_PARAMS); + logger.info("Full high_card_idx Response Content:\n{ {} }", Strings.toString(builder)); + for (int i = 0; i < totalHits; i++) { + SearchHit searchHit = response.getHits().getAt(i); + assertThat("Hit " + i + " with id: " + searchHit.getId(), searchHit.getIndex(), equalTo("high_card_idx")); + DocumentField hitField = searchHit.field(NUMBER_FIELD_NAME); + + assertThat("Hit " + i + " has wrong number of values", hitField.getValues().size(), equalTo(1)); + Long value = hitField.getValue(); + assertThat("Hit " + i + " has wrong value", value.intValue(), equalTo(i)); + } + assertThat(totalHits, equalTo(2000L)); + } + + private void updateGeohashBucketsCentroid(final GeoPoint location) { + String hash = Geohash.stringEncode(location.lon(), location.lat(), Geohash.PRECISION); + for (int precision = Geohash.PRECISION; precision > 0; --precision) { + final String h = hash.substring(0, precision); + expectedDocCountsForGeoHash.put(h, expectedDocCountsForGeoHash.getOrDefault(h, 0) + 1); + expectedCentroidsForGeoHash.put(h, updateHashCentroid(h, location)); + } + } + + private GeoPoint updateHashCentroid(String hash, final GeoPoint location) { + GeoPoint centroid = expectedCentroidsForGeoHash.getOrDefault(hash, null); + if (centroid == null) { + return new GeoPoint(location.lat(), location.lon()); + } + final int docCount = expectedDocCountsForGeoHash.get(hash); + final double newLon = centroid.lon() + (location.lon() - centroid.lon()) / docCount; + final double newLat = centroid.lat() + (location.lat() - centroid.lat()) / docCount; + return centroid.reset(newLat, newLon); + } + + private void updateBoundsBottomRight(GeoPoint geoPoint, GeoPoint currentBound) { + if (geoPoint.lat() < currentBound.lat()) { + currentBound.resetLat(geoPoint.lat()); + } + if (geoPoint.lon() > currentBound.lon()) { + currentBound.resetLon(geoPoint.lon()); + } + } + + private void updateBoundsTopLeft(GeoPoint geoPoint, GeoPoint currentBound) { + if (geoPoint.lat() > currentBound.lat()) { + currentBound.resetLat(geoPoint.lat()); + } + if (geoPoint.lon() < currentBound.lon()) { + currentBound.resetLon(geoPoint.lon()); + } + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoBoundsIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsIT.java similarity index 97% rename from server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoBoundsIT.java rename to modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsIT.java index 3af3b9e5212f8..5cbd98a4936e4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoBoundsIT.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsIT.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoPoint; @@ -43,21 +43,21 @@ import java.util.List; -import static org.opensearch.index.query.QueryBuilders.matchAllQuery; -import static org.opensearch.search.aggregations.AggregationBuilders.geoBounds; -import static org.opensearch.search.aggregations.AggregationBuilders.global; -import static org.opensearch.search.aggregations.AggregationBuilders.terms; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; -import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.closeTo; +import static org.opensearch.index.query.QueryBuilders.matchAllQuery; +import static org.opensearch.search.aggregations.AggregationBuilders.global; +import static org.opensearch.search.aggregations.AggregationBuilders.terms; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; +import static org.opensearch.geo.tests.common.AggregationBuilders.geoBounds; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class GeoBoundsIT extends AbstractGeoTestCase { +public class GeoBoundsIT extends AbstractGeoAggregatorTestCaseModulePlugin { private static final String aggName = "geoBounds"; public void testSingleValuedField() throws Exception { @@ -226,7 +226,8 @@ public void testSingleValuedFieldNearDateLineWrapLongitude() throws Exception { } /** - * This test forces the {@link GeoBoundsAggregator} to resize the {@link BigArray}s it uses to ensure they are resized correctly + * This test forces the {@link GeoBoundsAggregator} to resize the {@link BigArray}s it uses to ensure they are + * resized correctly */ public void testSingleValuedFieldAsSubAggToHighCardTermsAgg() { SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) diff --git a/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java b/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java index ed8c7c06fb3e8..64aac66b7eef3 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java +++ b/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java @@ -32,18 +32,36 @@ package org.opensearch.geo; +import org.opensearch.geo.search.aggregations.metrics.GeoBounds; +import org.opensearch.geo.search.aggregations.metrics.GeoBoundsAggregationBuilder; +import org.opensearch.geo.search.aggregations.metrics.InternalGeoBounds; import org.opensearch.index.mapper.GeoShapeFieldMapper; import org.opensearch.index.mapper.Mapper; import org.opensearch.plugins.MapperPlugin; import org.opensearch.plugins.Plugin; +import org.opensearch.plugins.SearchPlugin; import java.util.Collections; +import java.util.List; import java.util.Map; -public class GeoModulePlugin extends Plugin implements MapperPlugin { +public class GeoModulePlugin extends Plugin implements MapperPlugin, SearchPlugin { @Override public Map getMappers() { return Collections.singletonMap(GeoShapeFieldMapper.CONTENT_TYPE, new GeoShapeFieldMapper.TypeParser()); } + + /** + * Registering {@link GeoBounds} aggregation on GeoPoint field. + */ + @Override + public List getAggregations() { + final AggregationSpec spec = new AggregationSpec( + GeoBoundsAggregationBuilder.NAME, + GeoBoundsAggregationBuilder::new, + GeoBoundsAggregationBuilder.PARSER + ).addResultReader(InternalGeoBounds::new).setAggregatorRegistrar(GeoBoundsAggregationBuilder::registerAggregators); + return Collections.singletonList(spec); + } } diff --git a/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoBoundsAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoBoundsAggregator.java new file mode 100644 index 0000000000000..4a39fa1da04eb --- /dev/null +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoBoundsAggregator.java @@ -0,0 +1,128 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.metrics; + +import org.opensearch.common.lease.Releasables; +import org.opensearch.common.util.BigArrays; +import org.opensearch.common.util.DoubleArray; +import org.opensearch.search.aggregations.Aggregator; +import org.opensearch.search.aggregations.InternalAggregation; +import org.opensearch.search.aggregations.metrics.MetricsAggregator; +import org.opensearch.search.aggregations.support.ValuesSource; +import org.opensearch.search.aggregations.support.ValuesSourceConfig; +import org.opensearch.search.internal.SearchContext; + +import java.io.IOException; +import java.util.Map; + +/** + * Abstract class for doing the {@link GeoBounds} Aggregation over fields of type geo_shape and geo_point. + * + * @param Class extending the {@link ValuesSource} which will provide the data on which aggregation will happen. + * @opensearch.internal + */ +public abstract class AbstractGeoBoundsAggregator extends MetricsAggregator { + + protected final T valuesSource; + protected final boolean wrapLongitude; + protected DoubleArray tops; + protected DoubleArray bottoms; + protected DoubleArray posLefts; + protected DoubleArray posRights; + protected DoubleArray negLefts; + protected DoubleArray negRights; + + @SuppressWarnings("unchecked") + protected AbstractGeoBoundsAggregator( + String name, + SearchContext searchContext, + Aggregator aggregator, + ValuesSourceConfig valuesSourceConfig, + boolean wrapLongitude, + Map metaData + ) throws IOException { + super(name, searchContext, aggregator, metaData); + this.wrapLongitude = wrapLongitude; + valuesSource = valuesSourceConfig.hasValues() ? (T) valuesSourceConfig.getValuesSource() : null; + + if (valuesSource != null) { + final BigArrays bigArrays = context.bigArrays(); + tops = bigArrays.newDoubleArray(1, false); + tops.fill(0, tops.size(), Double.NEGATIVE_INFINITY); + bottoms = bigArrays.newDoubleArray(1, false); + bottoms.fill(0, bottoms.size(), Double.POSITIVE_INFINITY); + posLefts = bigArrays.newDoubleArray(1, false); + posLefts.fill(0, posLefts.size(), Double.POSITIVE_INFINITY); + posRights = bigArrays.newDoubleArray(1, false); + posRights.fill(0, posRights.size(), Double.NEGATIVE_INFINITY); + negLefts = bigArrays.newDoubleArray(1, false); + negLefts.fill(0, negLefts.size(), Double.POSITIVE_INFINITY); + negRights = bigArrays.newDoubleArray(1, false); + negRights.fill(0, negRights.size(), Double.NEGATIVE_INFINITY); + } + } + + /** + * Build an empty aggregation. + */ + @Override + public InternalAggregation buildEmptyAggregation() { + return new InternalGeoBounds( + name, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + Double.POSITIVE_INFINITY, + Double.NEGATIVE_INFINITY, + Double.POSITIVE_INFINITY, + Double.NEGATIVE_INFINITY, + wrapLongitude, + metadata() + ); + } + + /** + * Build an aggregation for data that has been collected into owningBucketOrd. + */ + @Override + public InternalAggregation buildAggregation(long owningBucketOrdinal) throws IOException { + if (valuesSource == null) { + return buildEmptyAggregation(); + } + double top = tops.get(owningBucketOrdinal); + double bottom = bottoms.get(owningBucketOrdinal); + double posLeft = posLefts.get(owningBucketOrdinal); + double posRight = posRights.get(owningBucketOrdinal); + double negLeft = negLefts.get(owningBucketOrdinal); + double negRight = negRights.get(owningBucketOrdinal); + return new InternalGeoBounds(name, top, bottom, posLeft, posRight, negLeft, negRight, wrapLongitude, metadata()); + } + + @Override + public void doClose() { + Releasables.close(tops, bottoms, posLefts, posRights, negLefts, negRights); + } + + protected void setBucketSize(final long bucket, final BigArrays bigArrays) { + if (bucket >= tops.size()) { + long from = tops.size(); + tops = bigArrays.grow(tops, bucket + 1); + tops.fill(from, tops.size(), Double.NEGATIVE_INFINITY); + bottoms = bigArrays.resize(bottoms, tops.size()); + bottoms.fill(from, bottoms.size(), Double.POSITIVE_INFINITY); + posLefts = bigArrays.resize(posLefts, tops.size()); + posLefts.fill(from, posLefts.size(), Double.POSITIVE_INFINITY); + posRights = bigArrays.resize(posRights, tops.size()); + posRights.fill(from, posRights.size(), Double.NEGATIVE_INFINITY); + negLefts = bigArrays.resize(negLefts, tops.size()); + negLefts.fill(from, negLefts.size(), Double.POSITIVE_INFINITY); + negRights = bigArrays.resize(negRights, tops.size()); + negRights.fill(from, negRights.size(), Double.NEGATIVE_INFINITY); + } + } +} diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBounds.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBounds.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBounds.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBounds.java index 380fbce85ada7..81ef502dda130 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBounds.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBounds.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.common.geo.GeoPoint; import org.opensearch.search.aggregations.Aggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilder.java similarity index 93% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilder.java index 64e27fa7e13d1..b2c441f9a951c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilder.java @@ -30,8 +30,9 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; +import org.opensearch.common.ParseField; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.xcontent.ObjectParser; @@ -40,6 +41,7 @@ import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; +import org.opensearch.search.aggregations.metrics.GeoBoundsAggregatorSupplier; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.search.aggregations.support.ValuesSourceConfig; @@ -57,6 +59,7 @@ */ public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "geo_bounds"; + private static final ParseField WRAP_LONGITUDE_FIELD = new ParseField("wrap_longitude"); public static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey<>( NAME, GeoBoundsAggregatorSupplier.class @@ -68,7 +71,7 @@ public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder< ); static { ValuesSourceAggregationBuilder.declareFields(PARSER, false, false, false); - PARSER.declareBoolean(GeoBoundsAggregationBuilder::wrapLongitude, GeoBoundsAggregator.WRAP_LONGITUDE_FIELD); + PARSER.declareBoolean(GeoBoundsAggregationBuilder::wrapLongitude, WRAP_LONGITUDE_FIELD); } public static void registerAggregators(ValuesSourceRegistry.Builder builder) { @@ -121,13 +124,6 @@ public GeoBoundsAggregationBuilder wrapLongitude(boolean wrapLongitude) { return this; } - /** - * Get whether to wrap longitudes. - */ - public boolean wrapLongitude() { - return wrapLongitude; - } - @Override public BucketCardinality bucketCardinality() { return BucketCardinality.NONE; @@ -145,7 +141,7 @@ protected GeoBoundsAggregatorFactory innerBuild( @Override public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { - builder.field(GeoBoundsAggregator.WRAP_LONGITUDE_FIELD.getPreferredName(), wrapLongitude); + builder.field(WRAP_LONGITUDE_FIELD.getPreferredName(), wrapLongitude); return builder; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregator.java similarity index 51% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregator.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregator.java index 054e8d4cb1c6c..a6518ea702be6 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregator.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregator.java @@ -30,17 +30,13 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.apache.lucene.index.LeafReaderContext; -import org.opensearch.common.ParseField; import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.DoubleArray; import org.opensearch.index.fielddata.MultiGeoPointValues; import org.opensearch.search.aggregations.Aggregator; -import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.LeafBucketCollector; import org.opensearch.search.aggregations.LeafBucketCollectorBase; import org.opensearch.search.aggregations.support.ValuesSource; @@ -51,22 +47,11 @@ import java.util.Map; /** - * Aggregate all docs into a geographic bounds + * Aggregate all docs into a geographic bounds for field GeoPoint. * * @opensearch.internal */ -final class GeoBoundsAggregator extends MetricsAggregator { - - static final ParseField WRAP_LONGITUDE_FIELD = new ParseField("wrap_longitude"); - - private final ValuesSource.GeoPoint valuesSource; - private final boolean wrapLongitude; - DoubleArray tops; - DoubleArray bottoms; - DoubleArray posLefts; - DoubleArray posRights; - DoubleArray negLefts; - DoubleArray negRights; +final class GeoBoundsAggregator extends AbstractGeoBoundsAggregator { GeoBoundsAggregator( String name, @@ -76,25 +61,7 @@ final class GeoBoundsAggregator extends MetricsAggregator { boolean wrapLongitude, Map metadata ) throws IOException { - super(name, aggregationContext, parent, metadata); - // TODO: stop expecting nulls here - this.valuesSource = valuesSourceConfig.hasValues() ? (ValuesSource.GeoPoint) valuesSourceConfig.getValuesSource() : null; - this.wrapLongitude = wrapLongitude; - if (valuesSource != null) { - final BigArrays bigArrays = context.bigArrays(); - tops = bigArrays.newDoubleArray(1, false); - tops.fill(0, tops.size(), Double.NEGATIVE_INFINITY); - bottoms = bigArrays.newDoubleArray(1, false); - bottoms.fill(0, bottoms.size(), Double.POSITIVE_INFINITY); - posLefts = bigArrays.newDoubleArray(1, false); - posLefts.fill(0, posLefts.size(), Double.POSITIVE_INFINITY); - posRights = bigArrays.newDoubleArray(1, false); - posRights.fill(0, posRights.size(), Double.NEGATIVE_INFINITY); - negLefts = bigArrays.newDoubleArray(1, false); - negLefts.fill(0, negLefts.size(), Double.POSITIVE_INFINITY); - negRights = bigArrays.newDoubleArray(1, false); - negRights.fill(0, negRights.size(), Double.NEGATIVE_INFINITY); - } + super(name, aggregationContext, parent, valuesSourceConfig, wrapLongitude, metadata); } @Override @@ -107,25 +74,10 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCol return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { - if (bucket >= tops.size()) { - long from = tops.size(); - tops = bigArrays.grow(tops, bucket + 1); - tops.fill(from, tops.size(), Double.NEGATIVE_INFINITY); - bottoms = bigArrays.resize(bottoms, tops.size()); - bottoms.fill(from, bottoms.size(), Double.POSITIVE_INFINITY); - posLefts = bigArrays.resize(posLefts, tops.size()); - posLefts.fill(from, posLefts.size(), Double.POSITIVE_INFINITY); - posRights = bigArrays.resize(posRights, tops.size()); - posRights.fill(from, posRights.size(), Double.NEGATIVE_INFINITY); - negLefts = bigArrays.resize(negLefts, tops.size()); - negLefts.fill(from, negLefts.size(), Double.POSITIVE_INFINITY); - negRights = bigArrays.resize(negRights, tops.size()); - negRights.fill(from, negRights.size(), Double.NEGATIVE_INFINITY); - } + setBucketSize(bucket, bigArrays); if (values.advanceExact(doc)) { final int valuesCount = values.docValueCount(); - for (int i = 0; i < valuesCount; ++i) { GeoPoint value = values.nextValue(); double top = tops.get(bucket); @@ -163,38 +115,4 @@ public void collect(int doc, long bucket) throws IOException { } }; } - - @Override - public InternalAggregation buildAggregation(long owningBucketOrdinal) { - if (valuesSource == null) { - return buildEmptyAggregation(); - } - double top = tops.get(owningBucketOrdinal); - double bottom = bottoms.get(owningBucketOrdinal); - double posLeft = posLefts.get(owningBucketOrdinal); - double posRight = posRights.get(owningBucketOrdinal); - double negLeft = negLefts.get(owningBucketOrdinal); - double negRight = negRights.get(owningBucketOrdinal); - return new InternalGeoBounds(name, top, bottom, posLeft, posRight, negLeft, negRight, wrapLongitude, metadata()); - } - - @Override - public InternalAggregation buildEmptyAggregation() { - return new InternalGeoBounds( - name, - Double.NEGATIVE_INFINITY, - Double.POSITIVE_INFINITY, - Double.POSITIVE_INFINITY, - Double.NEGATIVE_INFINITY, - Double.POSITIVE_INFINITY, - Double.NEGATIVE_INFINITY, - wrapLongitude, - metadata() - ); - } - - @Override - public void doClose() { - Releasables.close(tops, bottoms, posLefts, posRights, negLefts, negRights); - } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorFactory.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorFactory.java index 2c6b75842b6f5..149e052b4db7d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorFactory.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.aggregations.Aggregator; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalGeoBounds.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBounds.java similarity index 99% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/InternalGeoBounds.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBounds.java index 87018242ee8df..7c708de88a49c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/InternalGeoBounds.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBounds.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoPoint; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/ParsedGeoBounds.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/ParsedGeoBounds.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/ParsedGeoBounds.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/ParsedGeoBounds.java index a482fcfdf08dd..7643ac9d9a010 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/ParsedGeoBounds.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/ParsedGeoBounds.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.common.Nullable; import org.opensearch.common.collect.Tuple; diff --git a/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilderTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilderTests.java new file mode 100644 index 0000000000000..49b455bbf389e --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregationBuilderTests.java @@ -0,0 +1,43 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.metrics; + +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.plugins.Plugin; +import org.opensearch.search.aggregations.BaseAggregationTestCase; + +import java.util.Collection; +import java.util.Collections; + +public class GeoBoundsAggregationBuilderTests extends BaseAggregationTestCase { + + /** + * This registers the GeoShape mapper with the Tests so that it can be used for testing the aggregation builders + * + * @return A Collection containing {@link GeoModulePlugin} + */ + protected Collection> getPlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } + + @Override + protected GeoBoundsAggregationBuilder createTestAggregatorBuilder() { + GeoBoundsAggregationBuilder factory = new GeoBoundsAggregationBuilder(randomAlphaOfLengthBetween(1, 20)); + String field = randomAlphaOfLengthBetween(3, 20); + factory.field(field); + if (randomBoolean()) { + factory.wrapLongitude(randomBoolean()); + } + if (randomBoolean()) { + factory.missing("0,0"); + } + return factory; + } + +} diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorTests.java similarity index 88% rename from server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorTests.java index 6440c62e58e18..ee7a3c7e3faa2 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsAggregatorTests.java @@ -30,26 +30,42 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.apache.lucene.document.Document; import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.tests.common.AggregationInspectionHelper; +import org.opensearch.geo.tests.common.RandomGeoGenerator; import org.opensearch.index.mapper.GeoPointFieldMapper; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.plugins.SearchPlugin; import org.opensearch.search.aggregations.AggregatorTestCase; -import org.opensearch.search.aggregations.support.AggregationInspectionHelper; -import org.opensearch.test.geo.RandomGeoGenerator; -import static org.opensearch.search.aggregations.metrics.InternalGeoBoundsTests.GEOHASH_TOLERANCE; +import java.util.Collections; +import java.util.List; + import static org.hamcrest.Matchers.closeTo; public class GeoBoundsAggregatorTests extends AggregatorTestCase { + public static final double GEOHASH_TOLERANCE = 1E-5D; + + /** + * Overriding the Search Plugins list with {@link GeoModulePlugin} so that the testcase will know that this plugin is + * to be loaded during the tests. + * @return List of {@link SearchPlugin} + */ + @Override + protected List getSearchPlugins() { + return Collections.singletonList(new GeoModulePlugin()); + } + public void testEmpty() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder("my_agg").field("field").wrapLongitude(false); diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalGeoBoundsTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBoundsTests.java similarity index 81% rename from server/src/test/java/org/opensearch/search/aggregations/metrics/InternalGeoBoundsTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBoundsTests.java index e3857efff5d4d..22915212ff415 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/InternalGeoBoundsTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/metrics/InternalGeoBoundsTests.java @@ -30,11 +30,18 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; +import org.opensearch.common.ParseField; +import org.opensearch.common.xcontent.ContextParser; +import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.ParsedAggregation; import org.opensearch.test.InternalAggregationTestCase; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -44,6 +51,30 @@ public class InternalGeoBoundsTests extends InternalAggregationTestCase { static final double GEOHASH_TOLERANCE = 1E-5D; + /** + * Overriding the method so that tests can get the aggregation specs for namedWriteable. + * + * @return GeoPlugin + */ + @Override + protected SearchPlugin registerPlugin() { + return new GeoModulePlugin(); + } + + /** + * Overriding with the {@link ParsedGeoBounds} so that it can be parsed. We need to do this as {@link GeoModulePlugin} + * is registering this Aggregation. + * + * @return a List of {@link NamedXContentRegistry.Entry} + */ + @Override + protected List getNamedXContents() { + final List namedXContents = new ArrayList<>(getDefaultNamedXContents()); + final ContextParser parser = (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c); + namedXContents.add(new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(GeoBoundsAggregationBuilder.NAME), parser)); + return namedXContents; + } + @Override protected InternalGeoBounds createTestInstance(String name, Map metadata) { // we occasionally want to test top = Double.NEGATIVE_INFINITY since this triggers empty xContent object diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java new file mode 100644 index 0000000000000..c1f27b71c326d --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java @@ -0,0 +1,21 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.tests.common; + +import org.opensearch.geo.search.aggregations.metrics.GeoBounds; +import org.opensearch.geo.search.aggregations.metrics.GeoBoundsAggregationBuilder; + +public class AggregationBuilders { + /** + * Create a new {@link GeoBounds} aggregation with the given name. + */ + public static GeoBoundsAggregationBuilder geoBounds(String name) { + return new GeoBoundsAggregationBuilder(name); + } +} diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java new file mode 100644 index 0000000000000..208187bf34a5c --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java @@ -0,0 +1,18 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.tests.common; + +import org.opensearch.geo.search.aggregations.metrics.InternalGeoBounds; + +public class AggregationInspectionHelper { + + public static boolean hasValue(InternalGeoBounds agg) { + return (agg.topLeft() == null && agg.bottomRight() == null) == false; + } +} diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java new file mode 100644 index 0000000000000..2cf32c36b97ec --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java @@ -0,0 +1,86 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.tests.common; + +import org.opensearch.common.geo.GeoPoint; + +import java.util.Random; + +/** + * Random geo generation utilities for randomized {@code geo_point} type testing + * does not depend on jts or spatial4j. Use RandomShapeGenerator to create random OGC compliant shapes. + * This is a copy of the file present in the server folder. We need to keep both as there are tests which are + * dependent on that file. + */ +public class RandomGeoGenerator { + + public static void randomPoint(Random r, double[] pt) { + final double[] min = { -180, -90 }; + final double[] max = { 180, 90 }; + randomPointIn(r, min[0], min[1], max[0], max[1], pt); + } + + public static void randomPointIn( + Random r, + final double minLon, + final double minLat, + final double maxLon, + final double maxLat, + double[] pt + ) { + assert pt != null && pt.length == 2; + + // normalize min and max + double[] min = { normalizeLongitude(minLon), normalizeLatitude(minLat) }; + double[] max = { normalizeLongitude(maxLon), normalizeLatitude(maxLat) }; + final double[] tMin = new double[2]; + final double[] tMax = new double[2]; + tMin[0] = Math.min(min[0], max[0]); + tMax[0] = Math.max(min[0], max[0]); + tMin[1] = Math.min(min[1], max[1]); + tMax[1] = Math.max(min[1], max[1]); + + pt[0] = tMin[0] + r.nextDouble() * (tMax[0] - tMin[0]); + pt[1] = tMin[1] + r.nextDouble() * (tMax[1] - tMin[1]); + } + + public static GeoPoint randomPoint(Random r) { + return randomPointIn(r, -180, -90, 180, 90); + } + + public static GeoPoint randomPointIn(Random r, final double minLon, final double minLat, final double maxLon, final double maxLat) { + double[] pt = new double[2]; + randomPointIn(r, minLon, minLat, maxLon, maxLat, pt); + return new GeoPoint(pt[1], pt[0]); + } + + /** Puts latitude in range of -90 to 90. */ + private static double normalizeLatitude(double latitude) { + if (latitude >= -90 && latitude <= 90) { + return latitude; // common case, and avoids slight double precision shifting + } + double off = Math.abs((latitude + 90) % 360); + return (off <= 180 ? off : 360 - off) - 90; + } + + /** Puts longitude in range of -180 to +180. */ + private static double normalizeLongitude(double longitude) { + if (longitude >= -180 && longitude <= 180) { + return longitude; // common case, and avoids slight double precision shifting + } + double off = (longitude + 180) % 360; + if (off < 0) { + return 180 + off; + } else if (off == 0 && longitude > 0) { + return 180; + } else { + return -180 + off; + } + } +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java index 7d3f06760882d..26bfe59618275 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/MissingValueIT.java @@ -39,7 +39,6 @@ import org.opensearch.search.aggregations.bucket.terms.Terms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregatorFactory.ExecutionMode; import org.opensearch.search.aggregations.metrics.Cardinality; -import org.opensearch.search.aggregations.metrics.GeoBounds; import org.opensearch.search.aggregations.metrics.GeoCentroid; import org.opensearch.search.aggregations.metrics.Percentiles; import org.opensearch.search.aggregations.metrics.Stats; @@ -47,7 +46,6 @@ import static org.opensearch.search.aggregations.AggregationBuilders.cardinality; import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; -import static org.opensearch.search.aggregations.AggregationBuilders.geoBounds; import static org.opensearch.search.aggregations.AggregationBuilders.geoCentroid; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.percentiles; @@ -213,28 +211,6 @@ public void testStats() { assertEquals(4, stats.getAvg(), 0); } - public void testUnmappedGeoBounds() { - SearchResponse response = client().prepareSearch("idx") - .addAggregation(geoBounds("bounds").field("non_existing_field").missing("2,1")) - .get(); - assertSearchResponse(response); - GeoBounds bounds = response.getAggregations().get("bounds"); - assertThat(bounds.bottomRight().lat(), closeTo(2.0, 1E-5)); - assertThat(bounds.bottomRight().lon(), closeTo(1.0, 1E-5)); - assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); - assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); - } - - public void testGeoBounds() { - SearchResponse response = client().prepareSearch("idx").addAggregation(geoBounds("bounds").field("location").missing("2,1")).get(); - assertSearchResponse(response); - GeoBounds bounds = response.getAggregations().get("bounds"); - assertThat(bounds.bottomRight().lat(), closeTo(1.0, 1E-5)); - assertThat(bounds.bottomRight().lon(), closeTo(2.0, 1E-5)); - assertThat(bounds.topLeft().lat(), closeTo(2.0, 1E-5)); - assertThat(bounds.topLeft().lon(), closeTo(1.0, 1E-5)); - } - public void testGeoCentroid() { SearchResponse response = client().prepareSearch("idx") .addAggregation(geoCentroid("centroid").field("location").missing("2,1")) diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java index 0995497eba50e..80e025a3651a8 100644 --- a/server/src/main/java/org/opensearch/search/SearchModule.java +++ b/server/src/main/java/org/opensearch/search/SearchModule.java @@ -185,12 +185,10 @@ import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder; import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; -import org.opensearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.opensearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; import org.opensearch.search.aggregations.metrics.InternalAvg; import org.opensearch.search.aggregations.metrics.InternalCardinality; import org.opensearch.search.aggregations.metrics.InternalExtendedStats; -import org.opensearch.search.aggregations.metrics.InternalGeoBounds; import org.opensearch.search.aggregations.metrics.InternalGeoCentroid; import org.opensearch.search.aggregations.metrics.InternalHDRPercentileRanks; import org.opensearch.search.aggregations.metrics.InternalHDRPercentiles; @@ -664,12 +662,6 @@ private ValuesSourceRegistry registerAggregations(List plugins) { .addResultReader(InternalTopHits::new), builder ); - registerAggregation( - new AggregationSpec(GeoBoundsAggregationBuilder.NAME, GeoBoundsAggregationBuilder::new, GeoBoundsAggregationBuilder.PARSER) - .addResultReader(InternalGeoBounds::new) - .setAggregatorRegistrar(GeoBoundsAggregationBuilder::registerAggregators), - builder - ); registerAggregation( new AggregationSpec( GeoCentroidAggregationBuilder.NAME, diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java b/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java index 01e0f95b0d750..382455093309d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java @@ -78,8 +78,6 @@ import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.opensearch.search.aggregations.metrics.ExtendedStats; import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; -import org.opensearch.search.aggregations.metrics.GeoBounds; -import org.opensearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.opensearch.search.aggregations.metrics.GeoCentroid; import org.opensearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; import org.opensearch.search.aggregations.metrics.Max; @@ -364,13 +362,6 @@ public static TopHitsAggregationBuilder topHits(String name) { return new TopHitsAggregationBuilder(name); } - /** - * Create a new {@link GeoBounds} aggregation with the given name. - */ - public static GeoBoundsAggregationBuilder geoBounds(String name) { - return new GeoBoundsAggregationBuilder(name); - } - /** * Create a new {@link GeoCentroid} aggregation with the given name. */ diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java b/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java index 8d036503d1330..f36c4620d5b33 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java @@ -54,7 +54,6 @@ import org.opensearch.search.aggregations.metrics.InternalAvg; import org.opensearch.search.aggregations.metrics.InternalCardinality; import org.opensearch.search.aggregations.metrics.InternalExtendedStats; -import org.opensearch.search.aggregations.metrics.InternalGeoBounds; import org.opensearch.search.aggregations.metrics.InternalGeoCentroid; import org.opensearch.search.aggregations.metrics.InternalHDRPercentileRanks; import org.opensearch.search.aggregations.metrics.InternalHDRPercentiles; @@ -191,10 +190,6 @@ public static boolean hasValue(InternalExtendedStats agg) { return agg.getCount() > 0; } - public static boolean hasValue(InternalGeoBounds agg) { - return (agg.topLeft() == null && agg.bottomRight() == null) == false; - } - public static boolean hasValue(InternalGeoCentroid agg) { return agg.centroid() != null && agg.count() > 0; } diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java index 421865013a28c..111ce23f8a0cb 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java @@ -80,7 +80,6 @@ import org.opensearch.search.aggregations.metrics.InternalSumTests; import org.opensearch.search.aggregations.metrics.InternalAvgTests; import org.opensearch.search.aggregations.metrics.InternalCardinalityTests; -import org.opensearch.search.aggregations.metrics.InternalGeoBoundsTests; import org.opensearch.search.aggregations.metrics.InternalGeoCentroidTests; import org.opensearch.search.aggregations.metrics.InternalHDRPercentilesRanksTests; import org.opensearch.search.aggregations.metrics.InternalHDRPercentilesTests; @@ -142,7 +141,6 @@ private static List> getAggsTests() { aggsTests.add(new InternalStatsBucketTests()); aggsTests.add(new InternalExtendedStatsTests()); aggsTests.add(new InternalExtendedStatsBucketTests()); - aggsTests.add(new InternalGeoBoundsTests()); aggsTests.add(new InternalGeoCentroidTests()); aggsTests.add(new InternalHistogramTests()); aggsTests.add(new InternalDateHistogramTests()); diff --git a/server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsTests.java b/server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsTests.java deleted file mode 100644 index e132426680fc8..0000000000000 --- a/server/src/test/java/org/opensearch/search/aggregations/metrics/GeoBoundsTests.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/* - * Modifications Copyright OpenSearch Contributors. See - * GitHub history for details. - */ - -package org.opensearch.search.aggregations.metrics; - -import org.opensearch.search.aggregations.BaseAggregationTestCase; - -public class GeoBoundsTests extends BaseAggregationTestCase { - - @Override - protected GeoBoundsAggregationBuilder createTestAggregatorBuilder() { - GeoBoundsAggregationBuilder factory = new GeoBoundsAggregationBuilder(randomAlphaOfLengthBetween(1, 20)); - String field = randomAlphaOfLengthBetween(3, 20); - factory.field(field); - if (randomBoolean()) { - factory.wrapLongitude(randomBoolean()); - } - if (randomBoolean()) { - factory.missing("0,0"); - } - return factory; - } - -} diff --git a/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java index f138de152a488..a4099d66de28e 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java @@ -117,7 +117,6 @@ import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder; import org.opensearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.opensearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder; -import org.opensearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.opensearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; import org.opensearch.search.aggregations.metrics.InternalHDRPercentileRanks; import org.opensearch.search.aggregations.metrics.InternalHDRPercentiles; @@ -129,7 +128,6 @@ import org.opensearch.search.aggregations.metrics.ParsedAvg; import org.opensearch.search.aggregations.metrics.ParsedCardinality; import org.opensearch.search.aggregations.metrics.ParsedExtendedStats; -import org.opensearch.search.aggregations.metrics.ParsedGeoBounds; import org.opensearch.search.aggregations.metrics.ParsedGeoCentroid; import org.opensearch.search.aggregations.metrics.ParsedHDRPercentileRanks; import org.opensearch.search.aggregations.metrics.ParsedHDRPercentiles; @@ -261,7 +259,6 @@ public ReduceContext forFinalReduction() { map.put(StatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedStatsBucket.fromXContent(p, (String) c)); map.put(ExtendedStatsAggregationBuilder.NAME, (p, c) -> ParsedExtendedStats.fromXContent(p, (String) c)); map.put(ExtendedStatsBucketPipelineAggregationBuilder.NAME, (p, c) -> ParsedExtendedStatsBucket.fromXContent(p, (String) c)); - map.put(GeoBoundsAggregationBuilder.NAME, (p, c) -> ParsedGeoBounds.fromXContent(p, (String) c)); map.put(GeoCentroidAggregationBuilder.NAME, (p, c) -> ParsedGeoCentroid.fromXContent(p, (String) c)); map.put(HistogramAggregationBuilder.NAME, (p, c) -> ParsedHistogram.fromXContent(p, (String) c)); map.put(DateHistogramAggregationBuilder.NAME, (p, c) -> ParsedDateHistogram.fromXContent(p, (String) c)); diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 8d8df2fec39f9..1ab7785b17f5e 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -2097,6 +2097,7 @@ protected Collection> getMockPlugins() { if (addMockGeoShapeFieldMapper()) { mocks.add(TestGeoShapeFieldMapperPlugin.class); } + return Collections.unmodifiableList(mocks); } From 1c3f034dfda74f657289eab9aba3d9242fd08470 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Wed, 10 Aug 2022 18:12:19 +0530 Subject: [PATCH 03/27] [Remote Store] Add validator that forces segment replication type before enabling remote store (#4175) * Add validator that forces segment replication type before enabling remote store Signed-off-by: Sachin Kale --- .../cluster/metadata/IndexMetadata.java | 26 +++++++++++++++++ .../opensearch/index/IndexSettingsTests.java | 28 +++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 6e39809ac1a34..7210de8c70f65 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -292,6 +292,32 @@ public Iterator> settings() { public static final Setting INDEX_REMOTE_STORE_ENABLED_SETTING = Setting.boolSetting( SETTING_REMOTE_STORE_ENABLED, false, + new Setting.Validator<>() { + + @Override + public void validate(final Boolean value) {} + + @Override + public void validate(final Boolean value, final Map, Object> settings) { + final Object replicationType = settings.get(INDEX_REPLICATION_TYPE_SETTING); + if (replicationType != ReplicationType.SEGMENT && value == true) { + throw new IllegalArgumentException( + "Settings " + + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey() + + " cannot be enabled when " + + INDEX_REPLICATION_TYPE_SETTING.getKey() + + " is set to " + + settings.get(INDEX_REPLICATION_TYPE_SETTING) + ); + } + } + + @Override + public Iterator> settings() { + final List> settings = Collections.singletonList(INDEX_REPLICATION_TYPE_SETTING); + return settings.iterator(); + } + }, Property.IndexScope, Property.Final ); diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index 90aa36253fb7f..5f969180e6b2a 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -42,6 +42,7 @@ import org.opensearch.common.unit.ByteSizeValue; import org.opensearch.common.unit.TimeValue; import org.opensearch.index.translog.Translog; +import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; @@ -854,4 +855,31 @@ public void testEnablingRemoteTranslogStoreFailsWhenRemoteSegmentDisabled() { iae.getMessage() ); } + + public void testEnablingRemoteStoreFailsWhenReplicationTypeIsDocument() { + Settings indexSettings = Settings.builder() + .put("index.replication.type", ReplicationType.DOCUMENT) + .put("index.remote_store.enabled", true) + .build(); + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexSettings) + ); + assertEquals( + "Settings index.remote_store.enabled cannot be enabled when index.replication.type is set to DOCUMENT", + iae.getMessage() + ); + } + + public void testEnablingRemoteStoreFailsWhenReplicationTypeIsDefault() { + Settings indexSettings = Settings.builder().put("index.remote_store.enabled", true).build(); + IllegalArgumentException iae = expectThrows( + IllegalArgumentException.class, + () -> IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexSettings) + ); + assertEquals( + "Settings index.remote_store.enabled cannot be enabled when index.replication.type is set to DOCUMENT", + iae.getMessage() + ); + } } From 7dad0635d7216c2d4440cf9b8ecb703602d3a322 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 10 Aug 2022 10:07:13 -0700 Subject: [PATCH 04/27] [Segment Replication] Fix OngoingSegmentReplications to key by allocation ID instead of DiscoveryNode. (#4182) * Fix OngoingSegmentReplications to key by allocation ID instead of DiscoveryNode. This change fixes segrep to work with multiple shards per node by keying ongoing replications on allocation ID. This also updates cancel methods to ensure state is properly cleared on shard cancel. Signed-off-by: Marc Handalian * Clean up cancel methods. Signed-off-by: Marc Handalian Signed-off-by: Marc Handalian --- .../replication/SegmentReplicationIT.java | 66 ++++++++++++++-- .../OngoingSegmentReplications.java | 79 ++++++++++++------- .../SegmentReplicationSourceHandler.java | 15 +++- .../OngoingSegmentReplicationsTests.java | 42 +++++++++- .../SegmentReplicationSourceHandlerTests.java | 4 + 5 files changed, 166 insertions(+), 40 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index a1cc0148dcdac..dae2fa04a3a7e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -111,6 +111,54 @@ public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception { } } + public void testMultipleShards() throws Exception { + Settings indexSettings = Settings.builder() + .put(super.indexSettings()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexModule.INDEX_QUERY_CACHE_ENABLED_SETTING.getKey(), false) + .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .build(); + final String nodeA = internalCluster().startNode(); + final String nodeB = internalCluster().startNode(); + createIndex(INDEX_NAME, indexSettings); + ensureGreen(INDEX_NAME); + + final int initialDocCount = scaledRandomIntBetween(1, 200); + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(initialDocCount); + waitForDocs(initialDocCount, indexer); + refresh(INDEX_NAME); + waitForReplicaUpdate(); + + assertHitCount(client(nodeA).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), initialDocCount); + assertHitCount(client(nodeB).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), initialDocCount); + + final int additionalDocCount = scaledRandomIntBetween(0, 200); + final int expectedHitCount = initialDocCount + additionalDocCount; + indexer.start(additionalDocCount); + waitForDocs(expectedHitCount, indexer); + + flushAndRefresh(INDEX_NAME); + waitForReplicaUpdate(); + assertHitCount(client(nodeA).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount); + assertHitCount(client(nodeB).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount); + + ensureGreen(INDEX_NAME); + assertSegmentStats(REPLICA_COUNT); + } + } + public void testReplicationAfterForceMerge() throws Exception { final String nodeA = internalCluster().startNode(); final String nodeB = internalCluster().startNode(); @@ -262,15 +310,17 @@ private void waitForReplicaUpdate() throws Exception { final Map> segmentListMap = segmentsByShardType(replicationGroupSegments); final List primaryShardSegmentsList = segmentListMap.get(true); final List replicaShardSegments = segmentListMap.get(false); - + // if we don't have any segments yet, proceed. final ShardSegments primaryShardSegments = primaryShardSegmentsList.stream().findFirst().get(); - final Map latestPrimarySegments = getLatestSegments(primaryShardSegments); - final Long latestPrimaryGen = latestPrimarySegments.values().stream().findFirst().map(Segment::getGeneration).get(); - for (ShardSegments shardSegments : replicaShardSegments) { - final boolean isReplicaCaughtUpToPrimary = shardSegments.getSegments() - .stream() - .anyMatch(segment -> segment.getGeneration() == latestPrimaryGen); - assertTrue(isReplicaCaughtUpToPrimary); + if (primaryShardSegments.getSegments().isEmpty() == false) { + final Map latestPrimarySegments = getLatestSegments(primaryShardSegments); + final Long latestPrimaryGen = latestPrimarySegments.values().stream().findFirst().map(Segment::getGeneration).get(); + for (ShardSegments shardSegments : replicaShardSegments) { + final boolean isReplicaCaughtUpToPrimary = shardSegments.getSegments() + .stream() + .anyMatch(segment -> segment.getGeneration() == latestPrimaryGen); + assertTrue(isReplicaCaughtUpToPrimary); + } } } }); diff --git a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java index a9b032c98b70f..dfebe5f7cabf2 100644 --- a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java +++ b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java @@ -24,7 +24,10 @@ import java.io.IOException; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.function.Predicate; +import java.util.stream.Collectors; /** * Manages references to ongoing segrep events on a node. @@ -38,7 +41,7 @@ class OngoingSegmentReplications { private final RecoverySettings recoverySettings; private final IndicesService indicesService; private final Map copyStateMap; - private final Map nodesToHandlers; + private final Map allocationIdToHandlers; /** * Constructor. @@ -50,7 +53,7 @@ class OngoingSegmentReplications { this.indicesService = indicesService; this.recoverySettings = recoverySettings; this.copyStateMap = Collections.synchronizedMap(new HashMap<>()); - this.nodesToHandlers = ConcurrentCollections.newConcurrentMap(); + this.allocationIdToHandlers = ConcurrentCollections.newConcurrentMap(); } /** @@ -96,8 +99,7 @@ synchronized CopyState getCachedCopyState(ReplicationCheckpoint checkpoint) thro * @param listener {@link ActionListener} that resolves when sending files is complete. */ void startSegmentCopy(GetSegmentFilesRequest request, ActionListener listener) { - final DiscoveryNode node = request.getTargetNode(); - final SegmentReplicationSourceHandler handler = nodesToHandlers.get(node); + final SegmentReplicationSourceHandler handler = allocationIdToHandlers.get(request.getTargetAllocationId()); if (handler != null) { if (handler.isReplicating()) { throw new OpenSearchException( @@ -108,7 +110,7 @@ void startSegmentCopy(GetSegmentFilesRequest request, ActionListener wrappedListener = ActionListener.runBefore(listener, () -> { - final SegmentReplicationSourceHandler sourceHandler = nodesToHandlers.remove(node); + final SegmentReplicationSourceHandler sourceHandler = allocationIdToHandlers.remove(request.getTargetAllocationId()); if (sourceHandler != null) { removeCopyState(sourceHandler.getCopyState()); } @@ -123,19 +125,6 @@ void startSegmentCopy(GetSegmentFilesRequest request, ActionListener handler.getCopyState().getShard().shardId().equals(shard.shardId()), reason); + } + + /** + * Cancel any ongoing replications for a given {@link DiscoveryNode} + * + * @param node {@link DiscoveryNode} node for which to cancel replication events. + */ + void cancelReplication(DiscoveryNode node) { + cancelHandlers(handler -> handler.getTargetNode().equals(node), "Node left"); + } /** @@ -186,19 +180,25 @@ boolean isInCopyStateMap(ReplicationCheckpoint replicationCheckpoint) { } int size() { - return nodesToHandlers.size(); + return allocationIdToHandlers.size(); } int cachedCopyStateSize() { return copyStateMap.size(); } - private SegmentReplicationSourceHandler createTargetHandler(DiscoveryNode node, CopyState copyState, FileChunkWriter fileChunkWriter) { + private SegmentReplicationSourceHandler createTargetHandler( + DiscoveryNode node, + CopyState copyState, + String allocationId, + FileChunkWriter fileChunkWriter + ) { return new SegmentReplicationSourceHandler( node, fileChunkWriter, copyState.getShard().getThreadPool(), copyState, + allocationId, Math.toIntExact(recoverySettings.getChunkSize().getBytes()), recoverySettings.getMaxConcurrentFileChunks() ); @@ -231,4 +231,23 @@ private synchronized void removeCopyState(CopyState copyState) { copyStateMap.remove(copyState.getRequestedReplicationCheckpoint()); } } + + /** + * Remove handlers from allocationIdToHandlers map based on a filter predicate. + * This will also decref the handler's CopyState reference. + */ + private void cancelHandlers(Predicate predicate, String reason) { + final List allocationIds = allocationIdToHandlers.values() + .stream() + .filter(predicate) + .map(SegmentReplicationSourceHandler::getAllocationId) + .collect(Collectors.toList()); + for (String allocationId : allocationIds) { + final SegmentReplicationSourceHandler handler = allocationIdToHandlers.remove(allocationId); + if (handler != null) { + handler.cancel(reason); + removeCopyState(handler.getCopyState()); + } + } + } } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java index 8911302a722f5..46bad7951a2e1 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java @@ -54,6 +54,8 @@ class SegmentReplicationSourceHandler { private final List resources = new CopyOnWriteArrayList<>(); private final Logger logger; private final AtomicBoolean isReplicating = new AtomicBoolean(); + private final DiscoveryNode targetNode; + private final String allocationId; /** * Constructor. @@ -70,9 +72,11 @@ class SegmentReplicationSourceHandler { FileChunkWriter writer, ThreadPool threadPool, CopyState copyState, + String allocationId, int fileChunkSizeInBytes, int maxConcurrentFileChunks ) { + this.targetNode = targetNode; this.shard = copyState.getShard(); this.logger = Loggers.getLogger( SegmentReplicationSourceHandler.class, @@ -89,6 +93,7 @@ class SegmentReplicationSourceHandler { fileChunkSizeInBytes, maxConcurrentFileChunks ); + this.allocationId = allocationId; this.copyState = copyState; } @@ -118,7 +123,7 @@ public synchronized void sendFiles(GetSegmentFilesRequest request, ActionListene logger.debug( "delaying replication of {} as it is not listed as assigned to target node {}", shard.shardId(), - request.getTargetNode() + targetNode ); throw new DelayRecoveryException("source node does not have the shard listed in its state as allocated on the node"); } @@ -175,4 +180,12 @@ CopyState getCopyState() { public boolean isReplicating() { return isReplicating.get(); } + + public DiscoveryNode getTargetNode() { + return targetNode; + } + + public String getAllocationId() { + return allocationId; + } } diff --git a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java index d42e75871a45a..38c55620e1223 100644 --- a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java @@ -155,6 +155,9 @@ public void testCancelReplication() throws IOException { } public void testMultipleReplicasUseSameCheckpoint() throws IOException { + IndexShard secondReplica = newShard(primary.shardId(), false); + recoverReplica(secondReplica, primary, true); + OngoingSegmentReplications replications = new OngoingSegmentReplications(mockIndicesService, recoverySettings); final CheckpointInfoRequest request = new CheckpointInfoRequest( 1L, @@ -172,7 +175,7 @@ public void testMultipleReplicasUseSameCheckpoint() throws IOException { final CheckpointInfoRequest secondRequest = new CheckpointInfoRequest( 1L, - replica.routingEntry().allocationId().getId(), + secondReplica.routingEntry().allocationId().getId(), replicaDiscoveryNode, testCheckpoint ); @@ -187,6 +190,7 @@ public void testMultipleReplicasUseSameCheckpoint() throws IOException { assertEquals(0, copyState.refCount()); assertEquals(0, replications.size()); assertEquals(0, replications.cachedCopyStateSize()); + closeShards(secondReplica); } public void testStartCopyWithoutPrepareStep() { @@ -272,4 +276,40 @@ public void onFailure(Exception e) { } }); } + + public void testCancelAllReplicationsForShard() throws IOException { + // This tests when primary has multiple ongoing replications. + IndexShard replica_2 = newShard(primary.shardId(), false); + recoverReplica(replica_2, primary, true); + + OngoingSegmentReplications replications = new OngoingSegmentReplications(mockIndicesService, recoverySettings); + final CheckpointInfoRequest request = new CheckpointInfoRequest( + 1L, + replica.routingEntry().allocationId().getId(), + primaryDiscoveryNode, + testCheckpoint + ); + + final CopyState copyState = replications.prepareForReplication(request, mock(FileChunkWriter.class)); + assertEquals(1, copyState.refCount()); + + final CheckpointInfoRequest secondRequest = new CheckpointInfoRequest( + 1L, + replica_2.routingEntry().allocationId().getId(), + replicaDiscoveryNode, + testCheckpoint + ); + replications.prepareForReplication(secondRequest, mock(FileChunkWriter.class)); + + assertEquals(2, copyState.refCount()); + assertEquals(2, replications.size()); + assertEquals(1, replications.cachedCopyStateSize()); + + // cancel the primary's ongoing replications. + replications.cancel(primary, "Test"); + assertEquals(0, copyState.refCount()); + assertEquals(0, replications.size()); + assertEquals(0, replications.cachedCopyStateSize()); + closeShards(replica_2); + } } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java index 535cd5974490f..2c52772649acc 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java @@ -66,6 +66,7 @@ public void testSendFiles() throws IOException { chunkWriter, threadPool, copyState, + primary.routingEntry().allocationId().getId(), 5000, 1 ); @@ -103,6 +104,7 @@ public void testSendFiles_emptyRequest() throws IOException { chunkWriter, threadPool, copyState, + primary.routingEntry().allocationId().getId(), 5000, 1 ); @@ -141,6 +143,7 @@ public void testSendFileFails() throws IOException { chunkWriter, threadPool, copyState, + primary.routingEntry().allocationId().getId(), 5000, 1 ); @@ -178,6 +181,7 @@ public void testReplicationAlreadyRunning() throws IOException { chunkWriter, threadPool, copyState, + primary.routingEntry().allocationId().getId(), 5000, 1 ); From 8ccedd89d979eaaaec45523b409d61d3e4806c51 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Fri, 12 Aug 2022 18:58:51 +0530 Subject: [PATCH 05/27] [Remote Store] Change remote_store setting validation message to make it more clear (#4199) * Change remote_store setting validation message to make it more clear Signed-off-by: Sachin Kale --- .../org/opensearch/cluster/metadata/IndexMetadata.java | 8 ++++---- .../java/org/opensearch/index/IndexSettingsTests.java | 10 ++-------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java index 7210de8c70f65..759891e88039b 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java @@ -302,12 +302,12 @@ public void validate(final Boolean value, final Map, Object> settings final Object replicationType = settings.get(INDEX_REPLICATION_TYPE_SETTING); if (replicationType != ReplicationType.SEGMENT && value == true) { throw new IllegalArgumentException( - "Settings " + "To enable " + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey() - + " cannot be enabled when " + + ", " + INDEX_REPLICATION_TYPE_SETTING.getKey() - + " is set to " - + settings.get(INDEX_REPLICATION_TYPE_SETTING) + + " should be set to " + + ReplicationType.SEGMENT ); } } diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java index 5f969180e6b2a..e02eac85beafb 100644 --- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java +++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java @@ -865,10 +865,7 @@ public void testEnablingRemoteStoreFailsWhenReplicationTypeIsDocument() { IllegalArgumentException.class, () -> IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexSettings) ); - assertEquals( - "Settings index.remote_store.enabled cannot be enabled when index.replication.type is set to DOCUMENT", - iae.getMessage() - ); + assertEquals("To enable index.remote_store.enabled, index.replication.type should be set to SEGMENT", iae.getMessage()); } public void testEnablingRemoteStoreFailsWhenReplicationTypeIsDefault() { @@ -877,9 +874,6 @@ public void testEnablingRemoteStoreFailsWhenReplicationTypeIsDefault() { IllegalArgumentException.class, () -> IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexSettings) ); - assertEquals( - "Settings index.remote_store.enabled cannot be enabled when index.replication.type is set to DOCUMENT", - iae.getMessage() - ); + assertEquals("To enable index.remote_store.enabled, index.replication.type should be set to SEGMENT", iae.getMessage()); } } From b9a64cbfeb4da0e62367a344aa629cc59c953eff Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Fri, 12 Aug 2022 10:08:34 -0700 Subject: [PATCH 06/27] [Bug] [Segment Replication] Update store metadata recovery diff logic to ignore missing files causing exception (#4185) * Update Store for segment replication dif Signed-off-by: Poojita Raj Signed-off-by: Poojita Raj * Update recoveryDiff logic to ingore missing files causing exception on replica during copy Signed-off-by: Suraj Singh * Address review comments Signed-off-by: Suraj Singh Signed-off-by: Poojita Raj Signed-off-by: Suraj Singh Co-authored-by: Poojita Raj --- .../replication/SegmentReplicationIT.java | 52 +++++++ .../org/opensearch/index/store/Store.java | 88 ++++++++--- .../replication/SegmentReplicationTarget.java | 2 +- .../SegmentReplicationTargetTests.java | 138 +++++++++++++++++- 4 files changed, 260 insertions(+), 20 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index dae2fa04a3a7e..96ac703b9837e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -15,6 +15,7 @@ import org.opensearch.action.admin.indices.segments.IndicesSegmentResponse; import org.opensearch.action.admin.indices.segments.IndicesSegmentsRequest; import org.opensearch.action.admin.indices.segments.ShardSegments; +import org.opensearch.action.support.WriteRequest; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; @@ -36,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; @@ -247,6 +249,56 @@ public void testStartReplicaAfterPrimaryIndexesDocs() throws Exception { assertSegmentStats(REPLICA_COUNT); } + public void testDeleteOperations() throws Exception { + final String nodeA = internalCluster().startNode(); + final String nodeB = internalCluster().startNode(); + + createIndex(INDEX_NAME); + ensureGreen(INDEX_NAME); + final int initialDocCount = scaledRandomIntBetween(0, 200); + try ( + BackgroundIndexer indexer = new BackgroundIndexer( + INDEX_NAME, + "_doc", + client(), + -1, + RandomizedTest.scaledRandomIntBetween(2, 5), + false, + random() + ) + ) { + indexer.start(initialDocCount); + waitForDocs(initialDocCount, indexer); + refresh(INDEX_NAME); + waitForReplicaUpdate(); + + // wait a short amount of time to give replication a chance to complete. + assertHitCount(client(nodeA).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), initialDocCount); + assertHitCount(client(nodeB).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), initialDocCount); + + final int additionalDocCount = scaledRandomIntBetween(0, 200); + final int expectedHitCount = initialDocCount + additionalDocCount; + indexer.start(additionalDocCount); + waitForDocs(expectedHitCount, indexer); + waitForReplicaUpdate(); + + assertHitCount(client(nodeA).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount); + assertHitCount(client(nodeB).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount); + + ensureGreen(INDEX_NAME); + + Set ids = indexer.getIds(); + String id = ids.toArray()[0].toString(); + client(nodeA).prepareDelete(INDEX_NAME, id).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + refresh(INDEX_NAME); + waitForReplicaUpdate(); + + assertHitCount(client(nodeA).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount - 1); + assertHitCount(client(nodeB).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount - 1); + } + } + private void assertSegmentStats(int numberOfReplicas) throws IOException { final IndicesSegmentResponse indicesSegmentResponse = client().admin().indices().segments(new IndicesSegmentsRequest()).actionGet(); diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index 6828ab7d91b2c..163717ad94c2c 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -110,8 +110,8 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -1102,6 +1102,30 @@ public Map asMap() { private static final String LIV_FILE_EXTENSION = "liv"; // lucene 5 delete file private static final String SEGMENT_INFO_EXTENSION = "si"; + /** + * Helper method used to group store files according to segment and commit. + * + * @see MetadataSnapshot#recoveryDiff(MetadataSnapshot) + * @see MetadataSnapshot#segmentReplicationDiff(MetadataSnapshot) + */ + private Iterable> getGroupedFilesIterable() { + final Map> perSegment = new HashMap<>(); + final List perCommitStoreFiles = new ArrayList<>(); + for (StoreFileMetadata meta : this) { + final String segmentId = IndexFileNames.parseSegmentName(meta.name()); + final String extension = IndexFileNames.getExtension(meta.name()); + if (IndexFileNames.SEGMENTS.equals(segmentId) + || DEL_FILE_EXTENSION.equals(extension) + || LIV_FILE_EXTENSION.equals(extension)) { + // only treat del files as per-commit files fnm files are generational but only for upgradable DV + perCommitStoreFiles.add(meta); + } else { + perSegment.computeIfAbsent(segmentId, k -> new ArrayList<>()).add(meta); + } + } + return Iterables.concat(perSegment.values(), Collections.singleton(perCommitStoreFiles)); + } + /** * Returns a diff between the two snapshots that can be used for recovery. The given snapshot is treated as the * recovery target and this snapshot as the source. The returned diff will hold a list of files that are: @@ -1139,23 +1163,8 @@ public RecoveryDiff recoveryDiff(MetadataSnapshot recoveryTargetSnapshot) { final List identical = new ArrayList<>(); final List different = new ArrayList<>(); final List missing = new ArrayList<>(); - final Map> perSegment = new HashMap<>(); - final List perCommitStoreFiles = new ArrayList<>(); - - for (StoreFileMetadata meta : this) { - final String segmentId = IndexFileNames.parseSegmentName(meta.name()); - final String extension = IndexFileNames.getExtension(meta.name()); - if (IndexFileNames.SEGMENTS.equals(segmentId) - || DEL_FILE_EXTENSION.equals(extension) - || LIV_FILE_EXTENSION.equals(extension)) { - // only treat del files as per-commit files fnm files are generational but only for upgradable DV - perCommitStoreFiles.add(meta); - } else { - perSegment.computeIfAbsent(segmentId, k -> new ArrayList<>()).add(meta); - } - } final ArrayList identicalFiles = new ArrayList<>(); - for (List segmentFiles : Iterables.concat(perSegment.values(), Collections.singleton(perCommitStoreFiles))) { + for (List segmentFiles : getGroupedFilesIterable()) { identicalFiles.clear(); boolean consistent = true; for (StoreFileMetadata meta : segmentFiles) { @@ -1190,6 +1199,51 @@ public RecoveryDiff recoveryDiff(MetadataSnapshot recoveryTargetSnapshot) { return recoveryDiff; } + /** + * Segment Replication method + * Returns a diff between the two snapshots that can be used for getting list of files to copy over to a replica for segment replication. The given snapshot is treated as the + * target and this snapshot as the source. The returned diff will hold a list of files that are: + *
    + *
  • identical: they exist in both snapshots and they can be considered the same ie. they don't need to be recovered
    • + *
  • different: they exist in both snapshots but their they are not identical
    • + *
  • missing: files that exist in the source but not in the target
    • + *
+ */ + public RecoveryDiff segmentReplicationDiff(MetadataSnapshot recoveryTargetSnapshot) { + final List identical = new ArrayList<>(); + final List different = new ArrayList<>(); + final List missing = new ArrayList<>(); + final ArrayList identicalFiles = new ArrayList<>(); + for (List segmentFiles : getGroupedFilesIterable()) { + identicalFiles.clear(); + boolean consistent = true; + for (StoreFileMetadata meta : segmentFiles) { + StoreFileMetadata storeFileMetadata = recoveryTargetSnapshot.get(meta.name()); + if (storeFileMetadata == null) { + // Do not consider missing files as inconsistent in SegRep as replicas may lag while primary updates + // documents and generate new files specific to a segment + missing.add(meta); + } else if (storeFileMetadata.isSame(meta) == false) { + consistent = false; + different.add(meta); + } else { + identicalFiles.add(meta); + } + } + if (consistent) { + identical.addAll(identicalFiles); + } else { + different.addAll(identicalFiles); + } + } + RecoveryDiff recoveryDiff = new RecoveryDiff( + Collections.unmodifiableList(identical), + Collections.unmodifiableList(different), + Collections.unmodifiableList(missing) + ); + return recoveryDiff; + } + /** * Returns the number of files in this snapshot */ diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 516cfa91a787b..73d9a2f805d75 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -154,7 +154,7 @@ private void getFiles(CheckpointInfoResponse checkpointInfo, StepListener storeMetadataSnapshots = generateStoreMetadataSnapshot(docCount); + + SegmentReplicationSource segrepSource = new SegmentReplicationSource() { + @Override + public void getCheckpointMetadata( + long replicationId, + ReplicationCheckpoint checkpoint, + ActionListener listener + ) { + listener.onResponse( + new CheckpointInfoResponse(checkpoint, storeMetadataSnapshots.get(1), buffer.toArrayCopy(), Set.of(PENDING_DELETE_FILE)) + ); + } + + @Override + public void getSegmentFiles( + long replicationId, + ReplicationCheckpoint checkpoint, + List filesToFetch, + Store store, + ActionListener listener + ) { + listener.onResponse(new GetSegmentFilesResponse(filesToFetch)); + } + }; + SegmentReplicationTargetService.SegmentReplicationListener segRepListener = mock( + SegmentReplicationTargetService.SegmentReplicationListener.class + ); + + segrepTarget = spy(new SegmentReplicationTarget(repCheckpoint, indexShard, segrepSource, segRepListener)); + when(segrepTarget.getMetadataSnapshot()).thenReturn(storeMetadataSnapshots.get(0)); + segrepTarget.startReplication(new ActionListener() { + @Override + public void onResponse(Void replicationResponse) { + logger.info("No error processing checkpoint info"); + } + + @Override + public void onFailure(Exception e) { + assert (e instanceof IllegalStateException); + } + }); + } + + /** + * Generates a list of Store.MetadataSnapshot with two elements where second snapshot has extra files due to delete + * operation. A list of snapshots is returned so that identical files have same checksum. + * @param docCount + * @return + * @throws IOException + */ + List generateStoreMetadataSnapshot(int docCount) throws IOException { + List docList = new ArrayList<>(); + for (int i = 0; i < docCount; i++) { + Document document = new Document(); + String text = new String(new char[] { (char) (97 + i), (char) (97 + i) }); + document.add(new StringField("id", "" + i, random().nextBoolean() ? Field.Store.YES : Field.Store.NO)); + document.add(new TextField("str", text, Field.Store.YES)); + docList.add(document); + } + long seed = random().nextLong(); + Random random = new Random(seed); + IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random)).setCodec(TestUtil.getDefaultCodec()); + iwc.setMergePolicy(NoMergePolicy.INSTANCE); + iwc.setUseCompoundFile(true); + final ShardId shardId = new ShardId("index", "_na_", 1); + Store store = new Store(shardId, INDEX_SETTINGS, StoreTests.newDirectory(random()), new DummyShardLock(shardId)); + IndexWriter writer = new IndexWriter(store.directory(), iwc); + for (Document d : docList) { + writer.addDocument(d); + } + writer.commit(); + Store.MetadataSnapshot storeMetadata = store.getMetadata(); + // Delete one document to generate .liv file + writer.deleteDocuments(new Term("id", Integer.toString(random().nextInt(docCount)))); + writer.commit(); + Store.MetadataSnapshot storeMetadataWithDeletes = store.getMetadata(); + deleteContent(store.directory()); + writer.close(); + store.close(); + return Arrays.asList(storeMetadata, storeMetadataWithDeletes); + } + + public static void deleteContent(Directory directory) throws IOException { + final String[] files = directory.listAll(); + final List exceptions = new ArrayList<>(); + for (String file : files) { + try { + directory.deleteFile(file); + } catch (NoSuchFileException | FileNotFoundException e) { + // ignore + } catch (IOException e) { + exceptions.add(e); + } + } + ExceptionsHelper.rethrowAndSuppress(exceptions); + } + @Override public void tearDown() throws Exception { super.tearDown(); From 5f2e66bfb11ebaea536ee62bc3764a848a6ab330 Mon Sep 17 00:00:00 2001 From: Sachin Kale Date: Sat, 13 Aug 2022 22:48:16 +0530 Subject: [PATCH 07/27] [Remote Store] Add RemoteSegmentStoreDirectory to interact with remote segment store (#4020) * Add RemoteSegmentStoreDirectory to interact with remote segment store Signed-off-by: Sachin Kale --- .../index/store/RemoteDirectory.java | 12 +- .../store/RemoteSegmentStoreDirectory.java | 372 ++++++++++++++++++ .../index/store/RemoteDirectoryTests.java | 20 + .../RemoteSegmentStoreDirectoryTests.java | 339 ++++++++++++++++ 4 files changed, 742 insertions(+), 1 deletion(-) create mode 100644 server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java create mode 100644 server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java index 855457f275122..62e2b12896411 100644 --- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java +++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java @@ -33,7 +33,7 @@ * * @opensearch.internal */ -public final class RemoteDirectory extends Directory { +public class RemoteDirectory extends Directory { private final BlobContainer blobContainer; @@ -50,6 +50,16 @@ public String[] listAll() throws IOException { return blobContainer.listBlobs().keySet().stream().sorted().toArray(String[]::new); } + /** + * Returns names of files with given prefix in this directory. + * @param filenamePrefix The prefix to match against file names in the directory + * @return A list of the matching filenames in the directory + * @throws IOException if there were any failures in reading from the blob container + */ + public Collection listFilesByPrefix(String filenamePrefix) throws IOException { + return blobContainer.listBlobsByPrefix(filenamePrefix).keySet(); + } + /** * Removes an existing file in the directory. * diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java new file mode 100644 index 0000000000000..d7d6b29d08bfc --- /dev/null +++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java @@ -0,0 +1,372 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FilterDirectory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.opensearch.common.UUIDs; + +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * A RemoteDirectory extension for remote segment store. We need to make sure we don't overwrite a segment file once uploaded. + * In order to prevent segment overwrite which can occur due to two primary nodes for the same shard at the same time, + * a unique suffix is added to the uploaded segment file. This class keeps track of filename of segments stored + * in remote segment store vs filename in local filesystem and provides the consistent Directory interface so that + * caller will be accessing segment files in the same way as {@code FSDirectory}. Apart from storing actual segment files, + * remote segment store also keeps track of refresh checkpoints as metadata in a separate path which is handled by + * another instance of {@code RemoteDirectory}. + * @opensearch.internal + */ +public final class RemoteSegmentStoreDirectory extends FilterDirectory { + /** + * Each segment file is uploaded with unique suffix. + * For example, _0.cfe in local filesystem will be uploaded to remote segment store as _0.cfe__gX7bNIIBrs0AUNsR2yEG + */ + public static final String SEGMENT_NAME_UUID_SEPARATOR = "__"; + + public static final MetadataFilenameUtils.MetadataFilenameComparator METADATA_FILENAME_COMPARATOR = + new MetadataFilenameUtils.MetadataFilenameComparator(); + + /** + * remoteDataDirectory is used to store segment files at path: cluster_UUID/index_UUID/shardId/segments/data + */ + private final RemoteDirectory remoteDataDirectory; + /** + * remoteMetadataDirectory is used to store metadata files at path: cluster_UUID/index_UUID/shardId/segments/metadata + */ + private final RemoteDirectory remoteMetadataDirectory; + + /** + * To prevent explosion of refresh metadata files, we replace refresh files for the given primary term and generation + * This is achieved by uploading refresh metadata file with the same UUID suffix. + */ + private String metadataFileUniqueSuffix; + + /** + * Keeps track of local segment filename to uploaded filename along with other attributes like checksum. + * This map acts as a cache layer for uploaded segment filenames which helps avoid calling listAll() each time. + * It is important to initialize this map on creation of RemoteSegmentStoreDirectory and update it on each upload and delete. + */ + private Map segmentsUploadedToRemoteStore; + + private static final Logger logger = LogManager.getLogger(RemoteSegmentStoreDirectory.class); + + public RemoteSegmentStoreDirectory(RemoteDirectory remoteDataDirectory, RemoteDirectory remoteMetadataDirectory) throws IOException { + super(remoteDataDirectory); + this.remoteDataDirectory = remoteDataDirectory; + this.remoteMetadataDirectory = remoteMetadataDirectory; + init(); + } + + /** + * Initializes the cache which keeps track of all the segment files uploaded to the remote segment store. + * As this cache is specific to an instance of RemoteSegmentStoreDirectory, it is possible that cache becomes stale + * if another instance of RemoteSegmentStoreDirectory is used to upload/delete segment files. + * It is caller's responsibility to call init() again to ensure that cache is properly updated. + * @throws IOException if there were any failures in reading the metadata file + */ + public void init() throws IOException { + this.metadataFileUniqueSuffix = UUIDs.base64UUID(); + this.segmentsUploadedToRemoteStore = new ConcurrentHashMap<>(readLatestMetadataFile()); + } + + /** + * Read the latest metadata file to get the list of segments uploaded to the remote segment store. + * We upload a metadata file per refresh, but it is not unique per refresh. Refresh metadata file is unique for a given commit. + * The format of refresh metadata filename is: refresh_metadata__PrimaryTerm__Generation__UUID + * Refresh metadata files keep track of active segments for the shard at the time of refresh. + * In order to get the list of segment files uploaded to the remote segment store, we need to read the latest metadata file. + * Each metadata file contains a map where + * Key is - Segment local filename and + * Value is - local filename::uploaded filename::checksum + * @return Map of segment filename to uploaded filename with checksum + * @throws IOException if there were any failures in reading the metadata file + */ + private Map readLatestMetadataFile() throws IOException { + Map segmentMetadataMap = new HashMap<>(); + + Collection metadataFiles = remoteMetadataDirectory.listFilesByPrefix(MetadataFilenameUtils.METADATA_PREFIX); + Optional latestMetadataFile = metadataFiles.stream().max(METADATA_FILENAME_COMPARATOR); + + if (latestMetadataFile.isPresent()) { + logger.info("Reading latest Metadata file {}", latestMetadataFile.get()); + segmentMetadataMap = readMetadataFile(latestMetadataFile.get()); + } else { + logger.info("No metadata file found, this can happen for new index with no data uploaded to remote segment store"); + } + + return segmentMetadataMap; + } + + private Map readMetadataFile(String metadataFilename) throws IOException { + try (IndexInput indexInput = remoteMetadataDirectory.openInput(metadataFilename, IOContext.DEFAULT)) { + Map segmentMetadata = indexInput.readMapOfStrings(); + return segmentMetadata.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> UploadedSegmentMetadata.fromString(entry.getValue()))); + } + } + + /** + * Metadata of a segment that is uploaded to remote segment store. + */ + static class UploadedSegmentMetadata { + private static final String SEPARATOR = "::"; + private final String originalFilename; + private final String uploadedFilename; + private final String checksum; + + UploadedSegmentMetadata(String originalFilename, String uploadedFilename, String checksum) { + this.originalFilename = originalFilename; + this.uploadedFilename = uploadedFilename; + this.checksum = checksum; + } + + @Override + public String toString() { + return String.join(SEPARATOR, originalFilename, uploadedFilename, checksum); + } + + public static UploadedSegmentMetadata fromString(String uploadedFilename) { + String[] values = uploadedFilename.split(SEPARATOR); + return new UploadedSegmentMetadata(values[0], values[1], values[2]); + } + } + + /** + * Contains utility methods that provide various parts of metadata filename along with comparator + * Each metadata filename is of format: PREFIX__PrimaryTerm__Generation__UUID + */ + static class MetadataFilenameUtils { + public static final String SEPARATOR = "__"; + public static final String METADATA_PREFIX = "metadata"; + + /** + * Comparator to sort the metadata filenames. The order of sorting is: Primary Term, Generation, UUID + * Even though UUID sort does not provide any info on recency, it provides a consistent way to sort the filenames. + */ + static class MetadataFilenameComparator implements Comparator { + @Override + public int compare(String first, String second) { + String[] firstTokens = first.split(SEPARATOR); + String[] secondTokens = second.split(SEPARATOR); + if (!firstTokens[0].equals(secondTokens[0])) { + return firstTokens[0].compareTo(secondTokens[0]); + } + long firstPrimaryTerm = getPrimaryTerm(firstTokens); + long secondPrimaryTerm = getPrimaryTerm(secondTokens); + if (firstPrimaryTerm != secondPrimaryTerm) { + return firstPrimaryTerm > secondPrimaryTerm ? 1 : -1; + } else { + long firstGeneration = getGeneration(firstTokens); + long secondGeneration = getGeneration(secondTokens); + if (firstGeneration != secondGeneration) { + return firstGeneration > secondGeneration ? 1 : -1; + } else { + return getUuid(firstTokens).compareTo(getUuid(secondTokens)); + } + } + } + } + + // Visible for testing + static String getMetadataFilename(long primaryTerm, long generation, String uuid) { + return String.join( + SEPARATOR, + METADATA_PREFIX, + Long.toString(primaryTerm), + Long.toString(generation, Character.MAX_RADIX), + uuid + ); + } + + // Visible for testing + static long getPrimaryTerm(String[] filenameTokens) { + return Long.parseLong(filenameTokens[1]); + } + + // Visible for testing + static long getGeneration(String[] filenameTokens) { + return Long.parseLong(filenameTokens[2], Character.MAX_RADIX); + } + + // Visible for testing + static String getUuid(String[] filenameTokens) { + return filenameTokens[3]; + } + } + + /** + * Returns list of all the segment files uploaded to remote segment store till the last refresh checkpoint. + * Any segment file that is uploaded without corresponding metadata file will not be visible as part of listAll(). + * We chose not to return cache entries for listAll as cache can have entries for stale segments as well. + * Even if we plan to delete stale segments from remote segment store, it will be a periodic operation. + * @return segment filenames stored in remote segment store + * @throws IOException if there were any failures in reading the metadata file + */ + @Override + public String[] listAll() throws IOException { + return readLatestMetadataFile().keySet().toArray(new String[0]); + } + + /** + * Delete segment file from remote segment store. + * @param name the name of an existing segment file in local filesystem. + * @throws IOException if the file exists but could not be deleted. + */ + @Override + public void deleteFile(String name) throws IOException { + String remoteFilename = getExistingRemoteFilename(name); + if (remoteFilename != null) { + remoteDataDirectory.deleteFile(remoteFilename); + segmentsUploadedToRemoteStore.remove(name); + } + } + + /** + * Returns the byte length of a segment file in the remote segment store. + * @param name the name of an existing segment file in local filesystem. + * @throws IOException in case of I/O error + * @throws NoSuchFileException if the file does not exist in the cache or remote segment store + */ + @Override + public long fileLength(String name) throws IOException { + String remoteFilename = getExistingRemoteFilename(name); + if (remoteFilename != null) { + return remoteDataDirectory.fileLength(remoteFilename); + } else { + throw new NoSuchFileException(name); + } + } + + /** + * Creates and returns a new instance of {@link RemoteIndexOutput} which will be used to copy files to the remote + * segment store. + * @param name the name of the file to create. + * @throws IOException in case of I/O error + */ + @Override + public IndexOutput createOutput(String name, IOContext context) throws IOException { + return remoteDataDirectory.createOutput(getNewRemoteSegmentFilename(name), context); + } + + /** + * Opens a stream for reading an existing file and returns {@link RemoteIndexInput} enclosing the stream. + * @param name the name of an existing file. + * @throws IOException in case of I/O error + * @throws NoSuchFileException if the file does not exist either in cache or remote segment store + */ + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + String remoteFilename = getExistingRemoteFilename(name); + if (remoteFilename != null) { + return remoteDataDirectory.openInput(remoteFilename, context); + } else { + throw new NoSuchFileException(name); + } + } + + /** + * Copies an existing src file from directory from to a non-existent file dest in this directory. + * Once the segment is uploaded to remote segment store, update the cache accordingly. + */ + @Override + public void copyFrom(Directory from, String src, String dest, IOContext context) throws IOException { + String remoteFilename = getNewRemoteSegmentFilename(dest); + remoteDataDirectory.copyFrom(from, src, remoteFilename, context); + String checksum = getChecksumOfLocalFile(from, src); + UploadedSegmentMetadata segmentMetadata = new UploadedSegmentMetadata(src, remoteFilename, checksum); + segmentsUploadedToRemoteStore.put(src, segmentMetadata); + } + + /** + * Checks if the file exists in the uploadedSegments cache and the checksum matches. + * It is important to match the checksum as the same segment filename can be used for different + * segments due to a concurrency issue. + * @param localFilename filename of segment stored in local filesystem + * @param checksum checksum of the segment file + * @return true if file exists in cache and checksum matches. + */ + public boolean containsFile(String localFilename, String checksum) { + return segmentsUploadedToRemoteStore.containsKey(localFilename) + && segmentsUploadedToRemoteStore.get(localFilename).checksum.equals(checksum); + } + + /** + * Upload metadata file + * @param segmentFiles segment files that are part of the shard at the time of the latest refresh + * @param storeDirectory instance of local directory to temporarily create metadata file before upload + * @param primaryTerm primary term to be used in the name of metadata file + * @param generation commit generation + * @throws IOException in case of I/O error while uploading the metadata file + */ + public void uploadMetadata(Collection segmentFiles, Directory storeDirectory, long primaryTerm, long generation) + throws IOException { + synchronized (this) { + String metadataFilename = MetadataFilenameUtils.getMetadataFilename(primaryTerm, generation, this.metadataFileUniqueSuffix); + IndexOutput indexOutput = storeDirectory.createOutput(metadataFilename, IOContext.DEFAULT); + Map uploadedSegments = new HashMap<>(); + for (String file : segmentFiles) { + if (segmentsUploadedToRemoteStore.containsKey(file)) { + uploadedSegments.put(file, segmentsUploadedToRemoteStore.get(file).toString()); + } else { + throw new NoSuchFileException(file); + } + } + indexOutput.writeMapOfStrings(uploadedSegments); + indexOutput.close(); + storeDirectory.sync(Collections.singleton(metadataFilename)); + remoteMetadataDirectory.copyFrom(storeDirectory, metadataFilename, metadataFilename, IOContext.DEFAULT); + storeDirectory.deleteFile(metadataFilename); + } + } + + private String getChecksumOfLocalFile(Directory directory, String file) throws IOException { + try (IndexInput indexInput = directory.openInput(file, IOContext.DEFAULT)) { + return Long.toString(CodecUtil.retrieveChecksum(indexInput)); + } + } + + private String getExistingRemoteFilename(String localFilename) { + if (segmentsUploadedToRemoteStore.containsKey(localFilename)) { + return segmentsUploadedToRemoteStore.get(localFilename).uploadedFilename; + } else { + return null; + } + } + + private String getNewRemoteSegmentFilename(String localFilename) { + return localFilename + SEGMENT_NAME_UUID_SEPARATOR + UUIDs.base64UUID(); + } + + private String getLocalSegmentFilename(String remoteFilename) { + return remoteFilename.split(SEGMENT_NAME_UUID_SEPARATOR)[0]; + } + + // Visible for testing + Map getSegmentsUploadedToRemoteStore() { + return this.segmentsUploadedToRemoteStore; + } +} diff --git a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java index 2ded77d2cecfd..97575248b4ad3 100644 --- a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java +++ b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java @@ -15,11 +15,13 @@ import org.opensearch.common.blobstore.BlobContainer; import org.opensearch.common.blobstore.BlobMetadata; import org.opensearch.common.blobstore.support.PlainBlobMetadata; +import org.opensearch.common.collect.Set; import org.opensearch.test.OpenSearchTestCase; import java.io.IOException; import java.io.InputStream; import java.nio.file.NoSuchFileException; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -67,6 +69,24 @@ public void testListAllException() throws IOException { assertThrows(IOException.class, () -> remoteDirectory.listAll()); } + public void testListFilesByPrefix() throws IOException { + Map fileNames = Stream.of("abc", "abd", "abe", "abf", "abg") + .collect(Collectors.toMap(filename -> filename, filename -> new PlainBlobMetadata(filename, 100))); + + when(blobContainer.listBlobsByPrefix("ab")).thenReturn(fileNames); + + Collection actualFileNames = remoteDirectory.listFilesByPrefix("ab"); + Collection expectedFileName = Set.of("abc", "abd", "abe", "abf", "abg"); + assertEquals(expectedFileName, actualFileNames); + } + + public void testListFilesByPrefixException() throws IOException { + when(blobContainer.listBlobsByPrefix("abc")).thenThrow(new IOException("Error reading blob store")); + + assertThrows(IOException.class, () -> remoteDirectory.listFilesByPrefix("abc")); + verify(blobContainer).listBlobsByPrefix("abc"); + } + public void testDeleteFile() throws IOException { remoteDirectory.deleteFile("segment_1"); diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java new file mode 100644 index 0000000000000..4eabfa74625f2 --- /dev/null +++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java @@ -0,0 +1,339 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.store; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.junit.Before; +import org.opensearch.common.collect.Set; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.nio.file.NoSuchFileException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.startsWith; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class RemoteSegmentStoreDirectoryTests extends OpenSearchTestCase { + private RemoteDirectory remoteDataDirectory; + private RemoteDirectory remoteMetadataDirectory; + + private RemoteSegmentStoreDirectory remoteSegmentStoreDirectory; + + @Before + public void setup() throws IOException { + remoteDataDirectory = mock(RemoteDirectory.class); + remoteMetadataDirectory = mock(RemoteDirectory.class); + + remoteSegmentStoreDirectory = new RemoteSegmentStoreDirectory(remoteDataDirectory, remoteMetadataDirectory); + } + + public void testUploadedSegmentMetadataToString() { + RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = new RemoteSegmentStoreDirectory.UploadedSegmentMetadata( + "abc", + "pqr", + "123456" + ); + assertEquals("abc::pqr::123456", metadata.toString()); + } + + public void testUploadedSegmentMetadataFromString() { + RemoteSegmentStoreDirectory.UploadedSegmentMetadata metadata = RemoteSegmentStoreDirectory.UploadedSegmentMetadata.fromString( + "_0.cfe::_0.cfe__uuidxyz::4567" + ); + assertEquals("_0.cfe::_0.cfe__uuidxyz::4567", metadata.toString()); + } + + public void testGetMetadataFilename() { + // Generation 23 is replaced by n due to radix 32 + assertEquals( + RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX + "__12__n__uuid1", + RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(12, 23, "uuid1") + ); + } + + public void testGetPrimaryTermGenerationUuid() { + String[] filenameTokens = "abc__12__n__uuid_xyz".split(RemoteSegmentStoreDirectory.MetadataFilenameUtils.SEPARATOR); + assertEquals(12, RemoteSegmentStoreDirectory.MetadataFilenameUtils.getPrimaryTerm(filenameTokens)); + assertEquals(23, RemoteSegmentStoreDirectory.MetadataFilenameUtils.getGeneration(filenameTokens)); + assertEquals("uuid_xyz", RemoteSegmentStoreDirectory.MetadataFilenameUtils.getUuid(filenameTokens)); + } + + public void testMetadataFilenameComparator() { + List metadataFilenames = new ArrayList<>( + List.of( + "abc__10__20__uuid1", + "abc__12__2__uuid2", + "pqr__1__1__uuid0", + "abc__3__n__uuid3", + "abc__10__8__uuid8", + "abc__3__a__uuid4", + "abc__3__a__uuid5" + ) + ); + metadataFilenames.sort(RemoteSegmentStoreDirectory.METADATA_FILENAME_COMPARATOR); + assertEquals( + List.of( + "abc__3__a__uuid4", + "abc__3__a__uuid5", + "abc__3__n__uuid3", + "abc__10__8__uuid8", + "abc__10__20__uuid1", + "abc__12__2__uuid2", + "pqr__1__1__uuid0" + ), + metadataFilenames + ); + } + + public void testInitException() throws IOException { + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenThrow( + new IOException("Error") + ); + + assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.init()); + } + + public void testInitNoMetadataFile() throws IOException { + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( + List.of() + ); + + remoteSegmentStoreDirectory.init(); + Map actualCache = remoteSegmentStoreDirectory + .getSegmentsUploadedToRemoteStore(); + + assertEquals(Set.of(), actualCache.keySet()); + } + + private Map getDummyMetadata(String prefix, int commitGeneration) { + Map metadata = new HashMap<>(); + metadata.put(prefix + ".cfe", prefix + ".cfe::" + prefix + ".cfe__qrt::" + randomIntBetween(1000, 5000)); + metadata.put(prefix + ".cfs", prefix + ".cfs::" + prefix + ".cfs__zxd::" + randomIntBetween(1000, 5000)); + metadata.put(prefix + ".si", prefix + ".si::" + prefix + ".si__yui::" + randomIntBetween(1000, 5000)); + metadata.put( + "segments_" + commitGeneration, + "segments_" + commitGeneration + "::segments_" + commitGeneration + "__exv::" + randomIntBetween(1000, 5000) + ); + return metadata; + } + + private void populateMetadata() throws IOException { + List metadataFiles = List.of("metadata__1__5__abc", "metadata__1__6__pqr", "metadata__2__1__zxv"); + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( + metadataFiles + ); + + IndexInput indexInput = mock(IndexInput.class); + Map dummyMetadata = getDummyMetadata("_0", 1); + when(indexInput.readMapOfStrings()).thenReturn(dummyMetadata); + when(remoteMetadataDirectory.openInput("metadata__2__1__zxv", IOContext.DEFAULT)).thenReturn(indexInput); + } + + public void testInit() throws IOException { + populateMetadata(); + + when(remoteMetadataDirectory.listFilesByPrefix(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX)).thenReturn( + List.of("metadata__1__5__abc", "metadata__1__6__pqr", "metadata__2__1__zxv") + ); + + remoteSegmentStoreDirectory.init(); + + Map actualCache = remoteSegmentStoreDirectory + .getSegmentsUploadedToRemoteStore(); + + assertEquals(Set.of("_0.cfe", "_0.cfs", "_0.si", "segments_1"), actualCache.keySet()); + } + + public void testListAll() throws IOException { + populateMetadata(); + + assertEquals(Set.of("_0.cfe", "_0.cfs", "_0.si", "segments_1"), Set.of(remoteSegmentStoreDirectory.listAll())); + } + + public void testDeleteFile() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Map uploadedSegments = remoteSegmentStoreDirectory + .getSegmentsUploadedToRemoteStore(); + + assertTrue(uploadedSegments.containsKey("_0.si")); + assertFalse(uploadedSegments.containsKey("_100.si")); + + remoteSegmentStoreDirectory.deleteFile("_0.si"); + remoteSegmentStoreDirectory.deleteFile("_100.si"); + + verify(remoteDataDirectory).deleteFile(startsWith("_0.si")); + verify(remoteDataDirectory, times(0)).deleteFile(startsWith("_100.si")); + assertFalse(uploadedSegments.containsKey("_0.si")); + } + + public void testDeleteFileException() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + doThrow(new IOException("Error")).when(remoteDataDirectory).deleteFile(any()); + assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.deleteFile("_0.si")); + } + + public void testFileLenght() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Map uploadedSegments = remoteSegmentStoreDirectory + .getSegmentsUploadedToRemoteStore(); + + assertTrue(uploadedSegments.containsKey("_0.si")); + + when(remoteDataDirectory.fileLength(startsWith("_0.si"))).thenReturn(1234L); + + assertEquals(1234L, remoteSegmentStoreDirectory.fileLength("_0.si")); + } + + public void testFileLenghtNoSuchFile() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Map uploadedSegments = remoteSegmentStoreDirectory + .getSegmentsUploadedToRemoteStore(); + + assertFalse(uploadedSegments.containsKey("_100.si")); + assertThrows(NoSuchFileException.class, () -> remoteSegmentStoreDirectory.fileLength("_100.si")); + } + + public void testCreateOutput() throws IOException { + IndexOutput indexOutput = mock(IndexOutput.class); + when(remoteDataDirectory.createOutput(startsWith("abc"), eq(IOContext.DEFAULT))).thenReturn(indexOutput); + + assertEquals(indexOutput, remoteSegmentStoreDirectory.createOutput("abc", IOContext.DEFAULT)); + } + + public void testCreateOutputException() { + when(remoteDataDirectory.createOutput(startsWith("abc"), eq(IOContext.DEFAULT))).thenThrow(new IOException("Error")); + + assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.createOutput("abc", IOContext.DEFAULT)); + } + + public void testOpenInput() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + IndexInput indexInput = mock(IndexInput.class); + when(remoteDataDirectory.openInput(startsWith("_0.si"), eq(IOContext.DEFAULT))).thenReturn(indexInput); + + assertEquals(indexInput, remoteSegmentStoreDirectory.openInput("_0.si", IOContext.DEFAULT)); + } + + public void testOpenInputNoSuchFile() { + assertThrows(NoSuchFileException.class, () -> remoteSegmentStoreDirectory.openInput("_0.si", IOContext.DEFAULT)); + } + + public void testOpenInputException() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + when(remoteDataDirectory.openInput(startsWith("_0.si"), eq(IOContext.DEFAULT))).thenThrow(new IOException("Error")); + + assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.openInput("_0.si", IOContext.DEFAULT)); + } + + public void testCopyFrom() throws IOException { + String filename = "_100.si"; + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Directory storeDirectory = LuceneTestCase.newDirectory(); + IndexOutput indexOutput = storeDirectory.createOutput(filename, IOContext.DEFAULT); + indexOutput.writeString("Hello World!"); + CodecUtil.writeFooter(indexOutput); + indexOutput.close(); + storeDirectory.sync(List.of(filename)); + + assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); + remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, filename, IOContext.DEFAULT); + assertTrue(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); + + storeDirectory.close(); + } + + public void testCopyFromException() throws IOException { + String filename = "_100.si"; + Directory storeDirectory = LuceneTestCase.newDirectory(); + assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); + doThrow(new IOException("Error")).when(remoteDataDirectory).copyFrom(storeDirectory, filename, filename, IOContext.DEFAULT); + + assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, filename, IOContext.DEFAULT)); + assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename)); + + storeDirectory.close(); + } + + public void testContainsFile() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + // This is not the correct way to add files but the other way is to open up access to fields in UploadedSegmentMetadata + Map uploadedSegmentMetadataMap = remoteSegmentStoreDirectory + .getSegmentsUploadedToRemoteStore(); + uploadedSegmentMetadataMap.put( + "_100.si", + new RemoteSegmentStoreDirectory.UploadedSegmentMetadata("_100.si", "_100.si__uuid1", "1234") + ); + + assertTrue(remoteSegmentStoreDirectory.containsFile("_100.si", "1234")); + assertFalse(remoteSegmentStoreDirectory.containsFile("_100.si", "2345")); + assertFalse(remoteSegmentStoreDirectory.containsFile("_200.si", "1234")); + } + + public void testUploadMetadataEmpty() throws IOException { + Directory storeDirectory = mock(Directory.class); + IndexOutput indexOutput = mock(IndexOutput.class); + when(storeDirectory.createOutput(startsWith("metadata__12__o"), eq(IOContext.DEFAULT))).thenReturn(indexOutput); + + Collection segmentFiles = List.of("s1", "s2", "s3"); + assertThrows(NoSuchFileException.class, () -> remoteSegmentStoreDirectory.uploadMetadata(segmentFiles, storeDirectory, 12L, 24L)); + } + + public void testUploadMetadataNonEmpty() throws IOException { + populateMetadata(); + remoteSegmentStoreDirectory.init(); + + Directory storeDirectory = mock(Directory.class); + IndexOutput indexOutput = mock(IndexOutput.class); + when(storeDirectory.createOutput(startsWith("metadata__12__o"), eq(IOContext.DEFAULT))).thenReturn(indexOutput); + + Collection segmentFiles = List.of("_0.si"); + remoteSegmentStoreDirectory.uploadMetadata(segmentFiles, storeDirectory, 12L, 24L); + + verify(remoteMetadataDirectory).copyFrom( + eq(storeDirectory), + startsWith("metadata__12__o"), + startsWith("metadata__12__o"), + eq(IOContext.DEFAULT) + ); + String metadataString = remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().get("_0.si").toString(); + verify(indexOutput).writeMapOfStrings(Map.of("_0.si", metadataString)); + } +} From 0dbbd7292b767fdd7a62280be5807152f7bef0e1 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Mon, 15 Aug 2022 12:18:14 -0400 Subject: [PATCH 08/27] Update Gradle to 7.5.1 (#4211) Signed-off-by: Andriy Redko Signed-off-by: Andriy Redko --- gradle/wrapper/gradle-wrapper.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 24c164f0f1e12..58e9a16f424db 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -11,7 +11,7 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionSha256Sum=97a52d145762adc241bad7fd18289bf7f6801e08ece6badf80402fe2b9f250b1 +distributionSha256Sum=db9c8211ed63f61f60292c69e80d89196f9eb36665e369e7f00ac4cc841c2219 From 46d7a47304610f6f6093228e7a0b614d3f839bd4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 Aug 2022 12:53:25 -0400 Subject: [PATCH 09/27] Bump com.diffplug.spotless from 6.9.0 to 6.9.1 (#4210) Bumps com.diffplug.spotless from 6.9.0 to 6.9.1. --- updated-dependencies: - dependency-name: com.diffplug.spotless dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index e0bb961ce14c2..ce5ea6cdd7e11 100644 --- a/build.gradle +++ b/build.gradle @@ -55,7 +55,7 @@ plugins { id 'lifecycle-base' id 'opensearch.docker-support' id 'opensearch.global-build-info' - id "com.diffplug.spotless" version "6.9.0" apply false + id "com.diffplug.spotless" version "6.9.1" apply false id "org.gradle.test-retry" version "1.4.0" apply false id "test-report-aggregation" id 'jacoco-report-aggregation' From 96bfd1fa320c2f7d54e22587774da7e750adfbf8 Mon Sep 17 00:00:00 2001 From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> Date: Mon, 15 Aug 2022 17:43:10 +0000 Subject: [PATCH 10/27] [Segment Replication] Adding PrimaryMode check before publishing checkpoint and processing a received checkpoint. (#4157) * Adding PrimaryMode check before publishing checkpoint. Signed-off-by: Rishikesh1159 * Applying spotless check Signed-off-by: Rishikesh1159 * Moving segrep specific tests to SegmentReplicationIndexShardTests. Signed-off-by: Rishikesh1159 * Adding logic and tests for rejecting checkpoints if shard is in PrimaryMode. Signed-off-by: Rishikesh1159 * Applying ./gradlew :server:spotlessApply. Signed-off-by: Rishikesh1159 * Applying ./gradlew :server:spotlessApply Signed-off-by: Rishikesh1159 * Changing log level to warn in shouldProcessCheckpoint() of IndexShard.java class. Signed-off-by: Rishikesh1159 * Removing unnecessary lazy logging in shouldProcessCheckpoint(). Signed-off-by: Rishikesh1159 Signed-off-by: Rishikesh1159 --- .../shard/CheckpointRefreshListener.java | 2 +- .../opensearch/index/shard/IndexShard.java | 4 ++ .../index/shard/IndexShardTests.java | 2 +- .../SegmentReplicationIndexShardTests.java | 43 +++++++++++++++++++ .../SegmentReplicationTargetServiceTests.java | 17 ++++++++ 5 files changed, 66 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java index ac6754bf6a74a..96d74bea85920 100644 --- a/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java @@ -40,7 +40,7 @@ public void beforeRefresh() throws IOException { @Override public void afterRefresh(boolean didRefresh) throws IOException { - if (didRefresh) { + if (didRefresh && shard.getReplicationTracker().isPrimaryMode()) { publisher.publish(shard); } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 53324c94c4b08..ac259f9430a3d 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -1434,6 +1434,10 @@ public final boolean shouldProcessCheckpoint(ReplicationCheckpoint requestCheckp logger.trace(() -> new ParameterizedMessage("Ignoring new replication checkpoint - shard is not started {}", state())); return false; } + if (getReplicationTracker().isPrimaryMode()) { + logger.warn("Ignoring new replication checkpoint - shard is in primaryMode and cannot receive any checkpoints."); + return false; + } ReplicationCheckpoint localCheckpoint = getLatestReplicationCheckpoint(); if (localCheckpoint.isAheadOf(requestCheckpoint)) { logger.trace( diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java index 57c2289c848ef..8c00ab97a46ea 100644 --- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java @@ -3522,7 +3522,7 @@ public void testCheckpointRefreshListenerWithNull() throws IOException { } /** - * creates a new initializing shard. The shard will will be put in its proper path under the + * creates a new initializing shard. The shard will be put in its proper path under the * current node id the shard is assigned to. * @param checkpointPublisher Segment Replication Checkpoint Publisher to publish checkpoint */ diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 4f2784db93df2..d10f8ced963b7 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -14,11 +14,17 @@ import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; +import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationType; import java.io.IOException; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; + public class SegmentReplicationIndexShardTests extends OpenSearchIndexLevelReplicationTestCase { private static final Settings settings = Settings.builder() @@ -80,4 +86,41 @@ public void testIgnoreShardIdle() throws Exception { replica.awaitShardSearchActive(b -> assertFalse("A new RefreshListener should not be registered", b)); } } + + /** + * here we are starting a new primary shard in PrimaryMode and testing if the shard publishes checkpoint after refresh. + */ + public void testPublishCheckpointOnPrimaryMode() throws IOException { + final SegmentReplicationCheckpointPublisher mock = mock(SegmentReplicationCheckpointPublisher.class); + IndexShard shard = newStartedShard(true); + CheckpointRefreshListener refreshListener = new CheckpointRefreshListener(shard, mock); + refreshListener.afterRefresh(true); + + // verify checkpoint is published + verify(mock, times(1)).publish(any()); + closeShards(shard); + } + + /** + * here we are starting a new primary shard in PrimaryMode initially and starting relocation handoff. Later we complete relocation handoff then shard is no longer + * in PrimaryMode, and we test if the shard does not publish checkpoint after refresh. + */ + public void testPublishCheckpointAfterRelocationHandOff() throws IOException { + final SegmentReplicationCheckpointPublisher mock = mock(SegmentReplicationCheckpointPublisher.class); + IndexShard shard = newStartedShard(true); + CheckpointRefreshListener refreshListener = new CheckpointRefreshListener(shard, mock); + String id = shard.routingEntry().allocationId().getId(); + + // Starting relocation handoff + shard.getReplicationTracker().startRelocationHandoff(id); + + // Completing relocation handoff + shard.getReplicationTracker().completeRelocationHandoff(); + refreshListener.afterRefresh(true); + + // verify checkpoint is not published + verify(mock, times(0)).publish(any()); + closeShards(shard); + } + } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index b7cfda5dc1d64..004fa7f614ef1 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -208,6 +208,23 @@ public void testNewCheckpoint_validationPassesAndReplicationFails() throws IOExc closeShard(indexShard, false); } + /** + * here we are starting a new shard in PrimaryMode and testing that we don't process a checkpoint on shard when it is in PrimaryMode. + */ + public void testRejectCheckpointOnShardPrimaryMode() throws IOException { + SegmentReplicationTargetService spy = spy(sut); + + // Starting a new shard in PrimaryMode. + IndexShard primaryShard = newStartedShard(true); + IndexShard spyShard = spy(primaryShard); + doNothing().when(spy).startReplication(any(), any(), any()); + spy.onNewCheckpoint(aheadCheckpoint, spyShard); + + // Verify that checkpoint is not processed as shard is in PrimaryMode. + verify(spy, times(0)).startReplication(any(), any(), any()); + closeShards(primaryShard); + } + public void testReplicationOnDone() throws IOException { SegmentReplicationTargetService spy = spy(sut); IndexShard spyShard = spy(indexShard); From ea4cfcc1f1a4788982e8f7bb3175b0903226ca48 Mon Sep 17 00:00:00 2001 From: Yogev Mets Date: Mon, 15 Aug 2022 23:07:55 +0300 Subject: [PATCH 11/27] Set analyzer to regex query string search (#3967) Sets analyzer to regex query string search Signed-off-by: yyyogev --- .../test/search/190_index_prefix_search.yml | 22 +++++++++++++++++++ .../index/search/QueryStringQueryParser.java | 17 ++++++++------ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml index 6f276f669f815..ca8e89aa2d5b5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml @@ -17,6 +17,12 @@ setup: id: 1 body: { text: some short words with a stupendously long one } + - do: + index: + index: test + id: 2 + body: { text: sentence with UPPERCASE WORDS } + - do: indices.refresh: index: [test] @@ -76,6 +82,22 @@ setup: - match: {hits.max_score: 1} - match: {hits.hits.0._score: 1} +--- +"search with uppercase regex": + - do: + search: + rest_total_hits_as_int: true + index: test + body: + query: + query_string: + default_field: text + query: /UPPERCASE/ + + - match: {hits.total: 1} + - match: {hits.max_score: 1} + - match: {hits.hits.0._score: 1} + --- "search index prefixes with span_multi": - do: diff --git a/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java b/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java index cdb7464ff250a..6d59e861eb32f 100644 --- a/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java +++ b/server/src/main/java/org/opensearch/index/search/QueryStringQueryParser.java @@ -56,7 +56,6 @@ import org.apache.lucene.search.SynonymQuery; import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.automaton.RegExp; import org.opensearch.common.lucene.search.Queries; import org.opensearch.common.regex.Regex; import org.opensearch.common.unit.Fuzziness; @@ -565,7 +564,7 @@ private Query getPrefixQuerySingle(String field, String termStr) throws ParseExc if (currentFieldType == null || currentFieldType.getTextSearchInfo() == TextSearchInfo.NONE) { return newUnmappedFieldQuery(field); } - setAnalyzer(forceAnalyzer == null ? queryBuilder.context.getSearchAnalyzer(currentFieldType) : forceAnalyzer); + setAnalyzer(getSearchAnalyzer(currentFieldType)); Query query = null; if (currentFieldType.getTextSearchInfo().isTokenized() == false) { query = currentFieldType.prefixQuery(termStr, getMultiTermRewriteMethod(), context); @@ -741,6 +740,13 @@ private Query getWildcardQuerySingle(String field, String termStr) throws ParseE } } + private Analyzer getSearchAnalyzer(MappedFieldType currentFieldType) { + if (forceAnalyzer == null) { + return queryBuilder.context.getSearchAnalyzer(currentFieldType); + } + return forceAnalyzer; + } + @Override protected Query getRegexpQuery(String field, String termStr) throws ParseException { final int maxAllowedRegexLength = context.getIndexSettings().getMaxRegexLength(); @@ -781,11 +787,8 @@ private Query getRegexpQuerySingle(String field, String termStr) throws ParseExc if (currentFieldType == null) { return newUnmappedFieldQuery(field); } - if (forceAnalyzer != null) { - setAnalyzer(forceAnalyzer); - return super.getRegexpQuery(field, termStr); - } - return currentFieldType.regexpQuery(termStr, RegExp.ALL, 0, getDeterminizeWorkLimit(), getMultiTermRewriteMethod(), context); + setAnalyzer(getSearchAnalyzer(currentFieldType)); + return super.getRegexpQuery(field, termStr); } catch (RuntimeException e) { if (lenient) { return newLenientFieldQuery(field, e); From 411321e954cd695ec0c23193cc6b67828fdea8df Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Mon, 15 Aug 2022 15:09:46 -0500 Subject: [PATCH 12/27] [Upgrade] to lucene-9.4.0-snapshot-ddf0d0a (#4183) Upgrades to lucene-9.4.0-snapshot-ddf0d0a by refactoring to TermOrdValComparator as a top level class. Signed-off-by: Nicholas Walter Knize --- buildSrc/version.properties | 2 +- .../licenses/lucene-expressions-9.3.0.jar.sha1 | 1 - ...ucene-expressions-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + .../licenses/lucene-analysis-icu-9.3.0.jar.sha1 | 1 - ...cene-analysis-icu-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + .../licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 | 1 - ...analysis-kuromoji-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + .../licenses/lucene-analysis-nori-9.3.0.jar.sha1 | 1 - ...ene-analysis-nori-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + .../licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 | 1 - ...analysis-phonetic-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + .../licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 | 1 - ...-analysis-smartcn-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + .../licenses/lucene-analysis-stempel-9.3.0.jar.sha1 | 1 - ...-analysis-stempel-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + .../lucene-analysis-morfologik-9.3.0.jar.sha1 | 1 - ...alysis-morfologik-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + .../licenses/lucene-analysis-common-9.3.0.jar.sha1 | 1 - ...e-analysis-common-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + .../licenses/lucene-backward-codecs-9.3.0.jar.sha1 | 1 - ...e-backward-codecs-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/licenses/lucene-core-9.3.0.jar.sha1 | 1 - .../lucene-core-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/licenses/lucene-grouping-9.3.0.jar.sha1 | 1 - .../lucene-grouping-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/licenses/lucene-highlighter-9.3.0.jar.sha1 | 1 - ...ucene-highlighter-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/licenses/lucene-join-9.3.0.jar.sha1 | 1 - .../lucene-join-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/licenses/lucene-memory-9.3.0.jar.sha1 | 1 - .../lucene-memory-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/licenses/lucene-misc-9.3.0.jar.sha1 | 1 - .../lucene-misc-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/licenses/lucene-queries-9.3.0.jar.sha1 | 1 - .../lucene-queries-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/licenses/lucene-queryparser-9.3.0.jar.sha1 | 1 - ...ucene-queryparser-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/licenses/lucene-sandbox-9.3.0.jar.sha1 | 1 - .../lucene-sandbox-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/licenses/lucene-spatial-extras-9.3.0.jar.sha1 | 1 - ...ne-spatial-extras-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/licenses/lucene-spatial3d-9.3.0.jar.sha1 | 1 - .../lucene-spatial3d-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/licenses/lucene-suggest-9.3.0.jar.sha1 | 1 - .../lucene-suggest-9.4.0-snapshot-ddf0d0a.jar.sha1 | 1 + server/src/main/java/org/opensearch/Version.java | 2 +- .../java/org/opensearch/common/lucene/Lucene.java | 2 +- .../org/opensearch/index/codec/CodecService.java | 8 ++++---- .../codec/PerFieldMappingPostingFormatCodec.java | 4 ++-- .../BytesRefFieldComparatorSource.java | 11 ++++------- .../java/org/opensearch/index/codec/CodecTests.java | 12 ++++++------ .../index/engine/CompletionStatsCacheTests.java | 4 ++-- 52 files changed, 43 insertions(+), 46 deletions(-) delete mode 100644 modules/lang-expression/licenses/lucene-expressions-9.3.0.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analysis-icu-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analysis-nori-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-analysis-common-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-analysis-common-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-core-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-core-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-grouping-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-join-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-join-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-memory-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-memory-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-misc-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-misc-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-queries-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-queries-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-9.4.0-snapshot-ddf0d0a.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-9.3.0.jar.sha1 create mode 100644 server/licenses/lucene-suggest-9.4.0-snapshot-ddf0d0a.jar.sha1 diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 8e9911294977a..4af1acfed0ab2 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 3.0.0 -lucene = 9.3.0 +lucene = 9.4.0-snapshot-ddf0d0a bundled_jdk_vendor = adoptium bundled_jdk = 17.0.4+8 diff --git a/modules/lang-expression/licenses/lucene-expressions-9.3.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.3.0.jar.sha1 deleted file mode 100644 index 2d216277b3a8e..0000000000000 --- a/modules/lang-expression/licenses/lucene-expressions-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5583bcd3a24d3aae40b0a3152458021844ac09aa \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-9.4.0-snapshot-ddf0d0a.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..ec6906d730ac1 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +9f23e695b0c864fa9722e4f67d950266ca64d37b \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0.jar.sha1 deleted file mode 100644 index df4ae8d72dd2b..0000000000000 --- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -11dd9be0448fe594cf918f5260e193b3ab4e07a0 \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..83c10845cd35a --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +2f6cb0fd7387c6e0db3b86eef7d8677cea3e88a0 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 deleted file mode 100644 index 675bf726d2a65..0000000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -87c1357612f2f483174d1a63ea8c6680a1696bac \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..29387f38bc10c --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +6aff23715a2fba88d844ac83c61decce8ed480bd \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0.jar.sha1 deleted file mode 100644 index 8987f89c913df..0000000000000 --- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5d032dbeb3f4015741336a877dd4b0e62099246c \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..54b451abf5049 --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +f82d3eba195134f663865e9de3f511e16fbc7351 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 deleted file mode 100644 index 00d66c733c548..0000000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fe6ac8772b545e0abd0c755cd4bd07caad58edb9 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..87474064fbe0f --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +2af6e1996e696b1721a2ec7382bac9aa5096eecb \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 deleted file mode 100644 index 0c521b5f5ef6a..0000000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -288726e13b598c341e81aef8b5c9ce53f51889d0 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..6d35832a1a643 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +ec01d7f91f711abd75b539bb66a437db7cf1ca67 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0.jar.sha1 deleted file mode 100644 index ba98dd7e06f71..0000000000000 --- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -166d02f7f98f18c6607335030a404fcad8f57cd6 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..f93d1a153cd26 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +7041b3fa92b8687a84c4ce666b5718bbbc315db1 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1 deleted file mode 100644 index 88ac9a13e8ce3..0000000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3c0e4177aa87a4be2826a360f656f3559ea3f997 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.4.0-snapshot-ddf0d0a.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..77589a361badf --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +0a5ec9a237c2539e3cbabfadff707252e33b3599 \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.3.0.jar.sha1 b/server/licenses/lucene-analysis-common-9.3.0.jar.sha1 deleted file mode 100644 index 2e260eb028f4c..0000000000000 --- a/server/licenses/lucene-analysis-common-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -03496708a19a8a55a0dc4f61f8aa2febc6e8977c \ No newline at end of file diff --git a/server/licenses/lucene-analysis-common-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-analysis-common-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..0805590fd6efd --- /dev/null +++ b/server/licenses/lucene-analysis-common-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +3920c527fd5eee69e09f614391cef4e05c581c7f \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.3.0.jar.sha1 b/server/licenses/lucene-backward-codecs-9.3.0.jar.sha1 deleted file mode 100644 index 1dda17ee92fdb..0000000000000 --- a/server/licenses/lucene-backward-codecs-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -95ea01ee0d1e543e18e3cf58d8a6a27a587a7239 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-backward-codecs-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..a8c648e4c192a --- /dev/null +++ b/server/licenses/lucene-backward-codecs-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +d1dfcd42ea257355d5cbc64ac2f47142a550ae52 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.3.0.jar.sha1 b/server/licenses/lucene-core-9.3.0.jar.sha1 deleted file mode 100644 index fd870008c5bd4..0000000000000 --- a/server/licenses/lucene-core-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a030180999bc3f1a65f23f53b38098ca9daeee79 \ No newline at end of file diff --git a/server/licenses/lucene-core-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-core-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..779c9796ceae7 --- /dev/null +++ b/server/licenses/lucene-core-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +ae4757f88e97036b30eb1eac1d21da6dabc85a5e \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.3.0.jar.sha1 b/server/licenses/lucene-grouping-9.3.0.jar.sha1 deleted file mode 100644 index 6f63ca177d3c3..0000000000000 --- a/server/licenses/lucene-grouping-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -883071196e53ec93d2a53dcc8211ee30be6c00dc \ No newline at end of file diff --git a/server/licenses/lucene-grouping-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-grouping-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..9167482284f9d --- /dev/null +++ b/server/licenses/lucene-grouping-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +75485e3222b096027463378fe3bb2c8d1f529d25 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.3.0.jar.sha1 b/server/licenses/lucene-highlighter-9.3.0.jar.sha1 deleted file mode 100644 index 78264d8ee3713..0000000000000 --- a/server/licenses/lucene-highlighter-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -7e895c49b9991ea2ec08855c425b9eae44a08764 \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-highlighter-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..2090b009a57fe --- /dev/null +++ b/server/licenses/lucene-highlighter-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +d2a3d1f326f6d3bd6033b5620dc84f3c20a0412d \ No newline at end of file diff --git a/server/licenses/lucene-join-9.3.0.jar.sha1 b/server/licenses/lucene-join-9.3.0.jar.sha1 deleted file mode 100644 index 5e641f5f01075..0000000000000 --- a/server/licenses/lucene-join-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -04baaae4ce4a35ae919150dd17cd1e63b0da9d24 \ No newline at end of file diff --git a/server/licenses/lucene-join-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-join-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..df74fa911f7d2 --- /dev/null +++ b/server/licenses/lucene-join-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +2c6f6058c765a955e0544c6050aeee3a5e376e47 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.3.0.jar.sha1 b/server/licenses/lucene-memory-9.3.0.jar.sha1 deleted file mode 100644 index c8e86c7674ede..0000000000000 --- a/server/licenses/lucene-memory-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1a2203b332edc1366b9789f5286296e109dbc8c4 \ No newline at end of file diff --git a/server/licenses/lucene-memory-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-memory-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..3e9d65d73d36d --- /dev/null +++ b/server/licenses/lucene-memory-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +2a155679022106c7db356da32563580d8de043d7 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.3.0.jar.sha1 b/server/licenses/lucene-misc-9.3.0.jar.sha1 deleted file mode 100644 index 11a459a9f52ba..0000000000000 --- a/server/licenses/lucene-misc-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -61b502c9557247b6803a346c0bab20c9dc89d125 \ No newline at end of file diff --git a/server/licenses/lucene-misc-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-misc-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..f056cfe5b86ef --- /dev/null +++ b/server/licenses/lucene-misc-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +7c7ac2027a12bf02657ec3a421c8039736b98344 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.3.0.jar.sha1 b/server/licenses/lucene-queries-9.3.0.jar.sha1 deleted file mode 100644 index 2b577bd33b46a..0000000000000 --- a/server/licenses/lucene-queries-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -d8fe3bce3c05015c5fdb78279f36b9f1a75b98d8 \ No newline at end of file diff --git a/server/licenses/lucene-queries-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-queries-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..41ea4a342f949 --- /dev/null +++ b/server/licenses/lucene-queries-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +25978bb82b9f78537f0511f0be64253be19de6fd \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.3.0.jar.sha1 b/server/licenses/lucene-queryparser-9.3.0.jar.sha1 deleted file mode 100644 index b106860bf9f3e..0000000000000 --- a/server/licenses/lucene-queryparser-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -78f259a66d48f77a2d2b96a0a858efa08eba72dc \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-queryparser-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..e0687571df957 --- /dev/null +++ b/server/licenses/lucene-queryparser-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +e3b6ce41d5bd73fdcc80b5b2a40283c03525aa96 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.3.0.jar.sha1 b/server/licenses/lucene-sandbox-9.3.0.jar.sha1 deleted file mode 100644 index 82c2c6d85ca4c..0000000000000 --- a/server/licenses/lucene-sandbox-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -5ee318cf8e9a70c2c99e03e157465316a3d4a17a \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-sandbox-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..e03c731914757 --- /dev/null +++ b/server/licenses/lucene-sandbox-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +ae8649d2d01a416acdbe7c29f14b47a5594acb85 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.3.0.jar.sha1 b/server/licenses/lucene-spatial-extras-9.3.0.jar.sha1 deleted file mode 100644 index 8bbc5359487ff..0000000000000 --- a/server/licenses/lucene-spatial-extras-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c9b226b49ae987a4226791f023562187583eb9ad \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-spatial-extras-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..ea8b5cd1ddb1d --- /dev/null +++ b/server/licenses/lucene-spatial-extras-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +58049352bb5fc8683c389eb2eb879fb4f16ff9b3 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.3.0.jar.sha1 b/server/licenses/lucene-spatial3d-9.3.0.jar.sha1 deleted file mode 100644 index 31132ef0ad6df..0000000000000 --- a/server/licenses/lucene-spatial3d-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -201aa61856ae44fa494504591aed54fd9b75af16 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-spatial3d-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..a0c0dbe07af8f --- /dev/null +++ b/server/licenses/lucene-spatial3d-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +2d3a8f802e1bb439d945de81ba6b16d01b24d58a \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.3.0.jar.sha1 b/server/licenses/lucene-suggest-9.3.0.jar.sha1 deleted file mode 100644 index 71a263aa163f8..0000000000000 --- a/server/licenses/lucene-suggest-9.3.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fb5d7243ba67616edbda1ecf421c615dd595752d \ No newline at end of file diff --git a/server/licenses/lucene-suggest-9.4.0-snapshot-ddf0d0a.jar.sha1 b/server/licenses/lucene-suggest-9.4.0-snapshot-ddf0d0a.jar.sha1 new file mode 100644 index 0000000000000..1f332eac16c72 --- /dev/null +++ b/server/licenses/lucene-suggest-9.4.0-snapshot-ddf0d0a.jar.sha1 @@ -0,0 +1 @@ +11cdb21cf08feb19e074b4a101e1550dfd544379 \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java index 57f49f9c54591..27a39db1c50cb 100644 --- a/server/src/main/java/org/opensearch/Version.java +++ b/server/src/main/java/org/opensearch/Version.java @@ -96,7 +96,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_1_1 = new Version(2010199, org.apache.lucene.util.Version.LUCENE_9_2_0); public static final Version V_2_2_0 = new Version(2020099, org.apache.lucene.util.Version.LUCENE_9_3_0); public static final Version V_2_3_0 = new Version(2030099, org.apache.lucene.util.Version.LUCENE_9_3_0); - public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_3_0); + public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_4_0); public static final Version CURRENT = V_3_0_0; public static Version readVersion(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/opensearch/common/lucene/Lucene.java b/server/src/main/java/org/opensearch/common/lucene/Lucene.java index 74be762bfbcf9..2692a8fa2b914 100644 --- a/server/src/main/java/org/opensearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/opensearch/common/lucene/Lucene.java @@ -125,7 +125,7 @@ * @opensearch.internal */ public class Lucene { - public static final String LATEST_CODEC = "Lucene92"; + public static final String LATEST_CODEC = "Lucene94"; public static final String SOFT_DELETES_FIELD = "__soft_deletes"; diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index ff254a63fadb6..b1e73b3855759 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -34,8 +34,8 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; -import org.apache.lucene.codecs.lucene92.Lucene92Codec.Mode; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec.Mode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.mapper.MapperService; @@ -62,8 +62,8 @@ public class CodecService { public CodecService(@Nullable MapperService mapperService, Logger logger) { final MapBuilder codecs = MapBuilder.newMapBuilder(); if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene92Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene92Codec(Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Lucene94Codec()); + codecs.put(BEST_COMPRESSION_CODEC, new Lucene94Codec(Mode.BEST_COMPRESSION)); } else { codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger)); codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger)); diff --git a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java index fd0c66983208a..c101321e47350 100644 --- a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -36,7 +36,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; import org.opensearch.common.lucene.Lucene; import org.opensearch.index.mapper.CompletionFieldMapper; @@ -53,7 +53,7 @@ * * @opensearch.internal */ -public class PerFieldMappingPostingFormatCodec extends Lucene92Codec { +public class PerFieldMappingPostingFormatCodec extends Lucene94Codec { private final Logger logger; private final MapperService mapperService; private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat(); diff --git a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java index e816b366c3153..d7ce6ae8aba3e 100644 --- a/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java +++ b/server/src/main/java/org/opensearch/index/fielddata/fieldcomparator/BytesRefFieldComparatorSource.java @@ -40,6 +40,7 @@ import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.SortField; +import org.apache.lucene.search.comparators.TermOrdValComparator; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; import org.opensearch.common.util.BigArrays; @@ -99,7 +100,7 @@ public FieldComparator newComparator(String fieldname, int numHits, boolean e final boolean sortMissingLast = sortMissingLast(missingValue) ^ reversed; final BytesRef missingBytes = (BytesRef) missingObject(missingValue, reversed); if (indexFieldData instanceof IndexOrdinalsFieldData) { - return new FieldComparator.TermOrdValComparator(numHits, null, sortMissingLast) { + FieldComparator cmp = new TermOrdValComparator(numHits, indexFieldData.getFieldName(), sortMissingLast, reversed) { @Override protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException { @@ -121,13 +122,9 @@ protected SortedDocValues getSortedDocValues(LeafReaderContext context, String f return new ReplaceMissing(selectedValues, missingBytes); } } - - @Override - public void setScorer(Scorable scorer) { - BytesRefFieldComparatorSource.this.setScorer(scorer); - } - }; + cmp.disableSkipping(); + return cmp; } return new FieldComparator.TermValComparator(numHits, null, sortMissingLast) { diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index 0275066f9af1b..0a6338333bffc 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; @@ -65,21 +65,21 @@ public class CodecTests extends OpenSearchTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(); assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene92Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Lucene94Codec.class)); } public void testDefault() throws Exception { Codec codec = createCodecService().codec("default"); - assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_SPEED, codec); + assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_SPEED, codec); } public void testBestCompression() throws Exception { Codec codec = createCodecService().codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene92Codec.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_COMPRESSION, codec); } // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsCompressionEquals(Lucene94Codec.Mode expected, Codec actual) throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(null); iwc.setCodec(actual); @@ -91,7 +91,7 @@ private void assertStoredFieldsCompressionEquals(Lucene92Codec.Mode expected, Co SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Lucene92Codec.Mode.valueOf(v)); + assertEquals(expected, Lucene94Codec.Mode.valueOf(v)); ir.close(); dir.close(); } diff --git a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java index 340811352a203..575997dc2609e 100644 --- a/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java +++ b/server/src/test/java/org/opensearch/index/engine/CompletionStatsCacheTests.java @@ -32,7 +32,7 @@ package org.opensearch.index.engine; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene92.Lucene92Codec; +import org.apache.lucene.codecs.lucene94.Lucene94Codec; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; @@ -70,7 +70,7 @@ public void testExceptionsAreNotCached() { public void testCompletionStatsCache() throws IOException, InterruptedException { final IndexWriterConfig indexWriterConfig = newIndexWriterConfig(); final PostingsFormat postingsFormat = new Completion90PostingsFormat(); - indexWriterConfig.setCodec(new Lucene92Codec() { + indexWriterConfig.setCodec(new Lucene94Codec() { @Override public PostingsFormat getPostingsFormatForField(String field) { return postingsFormat; // all fields are suggest fields From a081f2f13e5df63dcc0897ded2154d7eab3f306a Mon Sep 17 00:00:00 2001 From: Pranav Garg Date: Tue, 16 Aug 2022 04:19:43 +0530 Subject: [PATCH 13/27] Adding documentation about creation of uber-JARs (#3785) * Adding documentation about creation of uber-JARs Signed-off-by: Pranav Garg * Fixing linelint error Signed-off-by: Pranav Garg * Comprehensive changes Signed-off-by: Pranav Garg * Adding PR changes Signed-off-by: Pranav Garg * PR changes Signed-off-by: Pranav Garg Signed-off-by: Pranav Garg Co-authored-by: Pranav Garg --- DEVELOPER_GUIDE.md | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index ce84d9658a808..8c2a6b4889122 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -4,7 +4,8 @@ - [Install Prerequisites](#install-prerequisites) - [JDK 11](#jdk-11) - [JDK 14](#jdk-14) - - [Runtime JDK](#runtime-jdk) + - [JDK 17](#jdk-17) + - [Custom Runtime JDK](#custom-runtime-jdk) - [Windows](#windows) - [Docker](#docker) - [Build](#build) @@ -12,6 +13,7 @@ - [Run OpenSearch](#run-opensearch) - [Use an Editor](#use-an-editor) - [IntelliJ IDEA](#intellij-idea) + - [Remote development using JetBrains Gateway](#remote-development-using-jetbrains-gateway) - [Visual Studio Code](#visual-studio-code) - [Eclipse](#eclipse) - [Project Layout](#project-layout) @@ -35,6 +37,7 @@ - [testImplementation](#testimplementation) - [Gradle Plugins](#gradle-plugins) - [Distribution Download Plugin](#distribution-download-plugin) + - [Creating fat-JAR of a Module](#creating-fat-jar-of-a-module) - [Misc](#misc) - [git-secrets](#git-secrets) - [Installation](#installation) @@ -49,7 +52,7 @@ - [Submitting Changes](#submitting-changes) - [Backports](#backports) - [LineLint](#linelint) - - [Lucene Snapshots](#lucene-snapshots) +- [Lucene Snapshots](#lucene-snapshots) # Developer Guide @@ -374,6 +377,42 @@ The Distribution Download plugin downloads the latest version of OpenSearch by d ./gradlew integTest -PcustomDistributionUrl="https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/1127/linux/x64/dist/opensearch-1.2.0-linux-x64.tar.gz" ``` +### Creating fat-JAR of a Module + +A fat-JAR (or an uber-JAR) is the JAR, which contains classes from all the libraries, on which your project depends and, of course, the classes of current project. + +There might be cases where a developer would like to add some custom logic to the code of a module (or multiple modules) and generate a fat-JAR that can be directly used by the dependency management tool. For example, in [#3665](https://github.com/opensearch-project/OpenSearch/pull/3665) a developer wanted to provide a tentative patch as a fat-JAR to a consumer for changes made in the high level REST client. + +Use [Gradle Shadow plugin](https://imperceptiblethoughts.com/shadow/). +Add the following to the `build.gradle` file of the module for which you want to create the fat-JAR, e.g. `client/rest-high-level/build.gradle`: + +``` +apply plugin: 'com.github.johnrengelman.shadow' +``` + +Run the `shadowJar` command using: +``` +./gradlew :client:rest-high-level:shadowJar +``` + +This will generate a fat-JAR in the `build/distributions` folder of the module, e.g. .`/client/rest-high-level/build/distributions/opensearch-rest-high-level-client-1.4.0-SNAPSHOT.jar`. + +You can further customize your fat-JAR by customising the plugin, More information about shadow plugin can be found [here](https://imperceptiblethoughts.com/shadow/). + +To use the generated JAR, install the JAR locally, e.g. +``` +mvn install:install-file -Dfile=src/main/resources/opensearch-rest-high-level-client-1.4.0-SNAPSHOT.jar -DgroupId=org.opensearch.client -DartifactId=opensearch-rest-high-level-client -Dversion=1.4.0-SNAPSHOT -Dpackaging=jar -DgeneratePom=true +``` + +Refer the installed JAR as any other maven artifact, e.g. + +``` + + org.opensearch.client + opensearch-rest-high-level-client + 1.4.0-SNAPSHOT + +``` ## Misc From 280b938b562fc2088a9afa8a8547f1adb96a84a5 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Mon, 15 Aug 2022 21:34:09 -0700 Subject: [PATCH 14/27] Segment Replication - Remove unnecessary call to markAllocationIdAsInSync. (#4224) This PR Removes an unnecessary call to markAllocationIdAsInSync on the primary shard when replication events complete. Recovery will manage this initial call. Signed-off-by: Marc Handalian Signed-off-by: Marc Handalian --- .../replication/SegmentReplicationSourceHandler.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java index 46bad7951a2e1..ce764900e433f 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java @@ -147,14 +147,6 @@ public synchronized void sendFiles(GetSegmentFilesRequest request, ActionListene transfer.start(); sendFileStep.whenComplete(r -> { - final String targetAllocationId = request.getTargetAllocationId(); - RunUnderPrimaryPermit.run( - () -> shard.markAllocationIdAsInSync(targetAllocationId, request.getCheckpoint().getSeqNo()), - shard.shardId() + " marking " + targetAllocationId + " as in sync", - shard, - cancellableThreads, - logger - ); try { future.onResponse(new GetSegmentFilesResponse(List.of(storeFileMetadata))); } finally { From 1e47b9528f9cb9b37ab84438467485c67ffe12ad Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Tue, 16 Aug 2022 09:01:42 -0500 Subject: [PATCH 15/27] [MUTE] search with uppercase regex in bwc testing (#4226) Prevents "search with uppercase regex" in 190_index_prefix_search.yml from running in bwc testing until feature is backported. Signed-off-by: Nicholas Walter Knize --- .../rest-api-spec/test/search/190_index_prefix_search.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml index ca8e89aa2d5b5..25d3dd160e031 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/190_index_prefix_search.yml @@ -84,6 +84,9 @@ setup: --- "search with uppercase regex": + - skip: + version: " - 2.99.99" + reason: uppercase regex not supported before 3.0.0 - do: search: rest_total_hits_as_int: true From 551f7c3481040a3b4b051a27eec973c90fb5def6 Mon Sep 17 00:00:00 2001 From: Bukhtawar Khan Date: Tue, 16 Aug 2022 22:18:56 +0530 Subject: [PATCH 16/27] Introduce TranslogFactory for Local/Remote Translog support (#4172) * Introduce TranslogFactory for Local/Remote Translog support Signed-off-by: Bukhtawar Khan --- .../opensearch/index/shard/IndexShardIT.java | 2 + .../org/opensearch/index/IndexService.java | 3 ++ .../opensearch/index/engine/EngineConfig.java | 19 ++++++++- .../index/engine/EngineConfigFactory.java | 7 +++- .../index/engine/InternalEngine.java | 3 +- .../index/engine/NRTReplicationEngine.java | 3 +- .../opensearch/index/engine/NoOpEngine.java | 17 ++++---- .../index/engine/ReadOnlyEngine.java | 17 ++++---- .../opensearch/index/shard/IndexShard.java | 7 +++- .../translog/InternalTranslogFactory.java | 41 +++++++++++++++++++ .../translog/InternalTranslogManager.java | 10 +++-- .../index/translog/TranslogFactory.java | 32 +++++++++++++++ .../translog/WriteOnlyTranslogManager.java | 6 ++- .../engine/EngineConfigFactoryTests.java | 7 +++- .../index/engine/EngineConfigTests.java | 4 +- .../InternalTranslogManagerTests.java | 21 ++++++---- .../index/shard/IndexShardTestCase.java | 2 + 17 files changed, 162 insertions(+), 39 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/translog/InternalTranslogFactory.java create mode 100644 server/src/main/java/org/opensearch/index/translog/TranslogFactory.java diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java index 2bf73b34247b3..3d8da7eac7690 100644 --- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java @@ -78,6 +78,7 @@ import org.opensearch.index.mapper.SourceToParse; import org.opensearch.index.seqno.RetentionLeaseSyncer; import org.opensearch.index.seqno.SequenceNumbers; +import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.TestTranslog; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogStats; @@ -675,6 +676,7 @@ public static final IndexShard newIndexShard( () -> {}, RetentionLeaseSyncer.EMPTY, cbs, + new InternalTranslogFactory(), SegmentReplicationCheckpointPublisher.EMPTY, null ); diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 210df9d342cb7..38065bf5aed20 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -88,6 +88,7 @@ import org.opensearch.index.shard.ShardPath; import org.opensearch.index.similarity.SimilarityService; import org.opensearch.index.store.Store; +import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.Translog; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.indices.cluster.IndicesClusterStateService; @@ -547,6 +548,8 @@ public synchronized IndexShard createShard( () -> globalCheckpointSyncer.accept(shardId), retentionLeaseSyncer, circuitBreakerService, + // TODO Replace with remote translog factory in the follow up PR + this.indexSettings.isRemoteTranslogStoreEnabled() ? null : new InternalTranslogFactory(), this.indexSettings.isSegRepEnabled() && routing.primary() ? checkpointPublisher : null, remoteStore ); diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java index 4ae6646ed14f0..ba30103f70269 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java @@ -51,8 +51,10 @@ import org.opensearch.index.seqno.RetentionLeases; import org.opensearch.index.shard.ShardId; import org.opensearch.index.store.Store; +import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.TranslogConfig; import org.opensearch.index.translog.TranslogDeletionPolicyFactory; +import org.opensearch.index.translog.TranslogFactory; import org.opensearch.indices.IndexingMemoryController; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.threadpool.ThreadPool; @@ -150,6 +152,8 @@ public Supplier retentionLeasesSupplier() { private final TranslogConfig translogConfig; + private final TranslogFactory translogFactory; + public EngineConfig( ShardId shardId, ThreadPool threadPool, @@ -253,7 +257,8 @@ public EngineConfig( retentionLeasesSupplier, primaryTermSupplier, tombstoneDocSupplier, - false + false, + new InternalTranslogFactory() ); } @@ -284,7 +289,8 @@ public EngineConfig( Supplier retentionLeasesSupplier, LongSupplier primaryTermSupplier, TombstoneDocSupplier tombstoneDocSupplier, - boolean isReadOnlyReplica + boolean isReadOnlyReplica, + TranslogFactory translogFactory ) { if (isReadOnlyReplica && indexSettings.isSegRepEnabled() == false) { throw new IllegalArgumentException("Shard can only be wired as a read only replica with Segment Replication enabled"); @@ -328,6 +334,7 @@ public EngineConfig( this.primaryTermSupplier = primaryTermSupplier; this.tombstoneDocSupplier = tombstoneDocSupplier; this.isReadOnlyReplica = isReadOnlyReplica; + this.translogFactory = translogFactory; } /** @@ -532,6 +539,14 @@ public boolean isReadOnlyReplica() { return indexSettings.isSegRepEnabled() && isReadOnlyReplica; } + /** + * Returns the underlying translog factory + * @return the translog factory + */ + public TranslogFactory getTranslogFactory() { + return translogFactory; + } + /** * A supplier supplies tombstone documents which will be used in soft-update methods. * The returned document consists only _uid, _seqno, _term and _version fields; other metadata fields are excluded. diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java index c8aec3570f8b5..f0db086e47816 100644 --- a/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java +++ b/server/src/main/java/org/opensearch/index/engine/EngineConfigFactory.java @@ -28,6 +28,7 @@ import org.opensearch.index.store.Store; import org.opensearch.index.translog.TranslogConfig; import org.opensearch.index.translog.TranslogDeletionPolicyFactory; +import org.opensearch.index.translog.TranslogFactory; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.PluginsService; @@ -147,7 +148,8 @@ public EngineConfig newEngineConfig( Supplier retentionLeasesSupplier, LongSupplier primaryTermSupplier, EngineConfig.TombstoneDocSupplier tombstoneDocSupplier, - boolean isReadOnlyReplica + boolean isReadOnlyReplica, + TranslogFactory translogFactory ) { CodecService codecServiceToUse = codecService; if (codecService == null && this.codecServiceFactory != null) { @@ -178,7 +180,8 @@ public EngineConfig newEngineConfig( retentionLeasesSupplier, primaryTermSupplier, tombstoneDocSupplier, - isReadOnlyReplica + isReadOnlyReplica, + translogFactory ); } diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java index 7d90c2ad653be..16599141b1345 100644 --- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java @@ -289,7 +289,8 @@ public void onFailure(String reason, Exception ex) { () -> getLocalCheckpointTracker(), translogUUID, new CompositeTranslogEventListener(Arrays.asList(internalTranslogEventListener, translogEventListener), shardId), - this::ensureOpen + this::ensureOpen, + engineConfig.getTranslogFactory() ); this.translogManager = translogManagerRef; this.softDeletesPolicy = newSoftDeletesPolicy(); diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java index 1f9306cf51e1e..a481203ba3dea 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java @@ -101,7 +101,8 @@ public void onAfterTranslogSync() { } } }, - this + this, + engineConfig.getTranslogFactory() ); this.translogManager = translogManagerRef; } catch (IOException e) { diff --git a/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java b/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java index 4bc37084675ea..f6c5bf7640a73 100644 --- a/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NoOpEngine.java @@ -195,14 +195,15 @@ public void trimUnreferencedTranslogFiles() throws TranslogException { final TranslogDeletionPolicy translogDeletionPolicy = new DefaultTranslogDeletionPolicy(-1, -1, 0); translogDeletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpoint); try ( - Translog translog = new Translog( - translogConfig, - translogUuid, - translogDeletionPolicy, - engineConfig.getGlobalCheckpointSupplier(), - engineConfig.getPrimaryTermSupplier(), - seqNo -> {} - ) + Translog translog = engineConfig.getTranslogFactory() + .newTranslog( + translogConfig, + translogUuid, + translogDeletionPolicy, + engineConfig.getGlobalCheckpointSupplier(), + engineConfig.getPrimaryTermSupplier(), + seqNo -> {} + ) ) { translog.trimUnreferencedReaders(); // refresh the translog stats diff --git a/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java index cebe262fee5d1..f426768119c1d 100644 --- a/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/ReadOnlyEngine.java @@ -258,14 +258,15 @@ private static TranslogStats translogStats(final EngineConfig config, final Segm final long localCheckpoint = Long.parseLong(infos.getUserData().get(SequenceNumbers.LOCAL_CHECKPOINT_KEY)); translogDeletionPolicy.setLocalCheckpointOfSafeCommit(localCheckpoint); try ( - Translog translog = new Translog( - translogConfig, - translogUuid, - translogDeletionPolicy, - config.getGlobalCheckpointSupplier(), - config.getPrimaryTermSupplier(), - seqNo -> {} - ) + Translog translog = config.getTranslogFactory() + .newTranslog( + translogConfig, + translogUuid, + translogDeletionPolicy, + config.getGlobalCheckpointSupplier(), + config.getPrimaryTermSupplier(), + seqNo -> {} + ) ) { return translog.stats(); } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index ac259f9430a3d..8970e0734f184 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -151,6 +151,7 @@ import org.opensearch.index.store.StoreStats; import org.opensearch.index.translog.Translog; import org.opensearch.index.translog.TranslogConfig; +import org.opensearch.index.translog.TranslogFactory; import org.opensearch.index.translog.TranslogRecoveryRunner; import org.opensearch.index.translog.TranslogStats; import org.opensearch.index.warmer.ShardIndexWarmerService; @@ -308,6 +309,7 @@ Runnable getGlobalCheckpointSyncer() { private final ReferenceManager.RefreshListener checkpointRefreshListener; private final Store remoteStore; + private final TranslogFactory translogFactory; public IndexShard( final ShardRouting shardRouting, @@ -330,6 +332,7 @@ public IndexShard( final Runnable globalCheckpointSyncer, final RetentionLeaseSyncer retentionLeaseSyncer, final CircuitBreakerService circuitBreakerService, + final TranslogFactory translogFactory, @Nullable final SegmentReplicationCheckpointPublisher checkpointPublisher, @Nullable final Store remoteStore ) throws IOException { @@ -420,6 +423,7 @@ public boolean shouldCache(Query query) { this.checkpointRefreshListener = null; } this.remoteStore = remoteStore; + this.translogFactory = translogFactory; } public ThreadPool getThreadPool() { @@ -3254,7 +3258,8 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro replicationTracker::getRetentionLeases, () -> getOperationPrimaryTerm(), tombstoneDocSupplier(), - indexSettings.isSegRepEnabled() && shardRouting.primary() == false + indexSettings.isSegRepEnabled() && shardRouting.primary() == false, + translogFactory ); } diff --git a/server/src/main/java/org/opensearch/index/translog/InternalTranslogFactory.java b/server/src/main/java/org/opensearch/index/translog/InternalTranslogFactory.java new file mode 100644 index 0000000000000..566eda4fe4a6e --- /dev/null +++ b/server/src/main/java/org/opensearch/index/translog/InternalTranslogFactory.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.translog; + +import java.io.IOException; +import java.util.function.LongConsumer; +import java.util.function.LongSupplier; + +/** + * Translog Factory for the local on-disk {@link Translog} + * + * @opensearch.internal + */ +public class InternalTranslogFactory implements TranslogFactory { + + @Override + public Translog newTranslog( + TranslogConfig translogConfig, + String translogUUID, + TranslogDeletionPolicy translogDeletionPolicy, + LongSupplier globalCheckpointSupplier, + LongSupplier primaryTermSupplier, + LongConsumer persistedSequenceNumberConsumer + ) throws IOException { + + return new Translog( + translogConfig, + translogUUID, + translogDeletionPolicy, + globalCheckpointSupplier, + primaryTermSupplier, + persistedSequenceNumberConsumer + ); + } +} diff --git a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java index ac82cf246cc55..fd52e02132006 100644 --- a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java +++ b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java @@ -54,7 +54,8 @@ public InternalTranslogManager( Supplier localCheckpointTrackerSupplier, String translogUUID, TranslogEventListener translogEventListener, - LifecycleAware engineLifeCycleAware + LifecycleAware engineLifeCycleAware, + TranslogFactory translogFactory ) throws IOException { this.shardId = shardId; this.readLock = readLock; @@ -67,7 +68,7 @@ public InternalTranslogManager( if (tracker != null) { tracker.markSeqNoAsPersisted(seqNo); } - }, translogUUID); + }, translogUUID, translogFactory); assert translog.getGeneration() != null; this.translog = translog; assert pendingTranslogRecovery.get() == false : "translog recovery can't be pending before we set it"; @@ -333,10 +334,11 @@ protected Translog openTranslog( TranslogDeletionPolicy translogDeletionPolicy, LongSupplier globalCheckpointSupplier, LongConsumer persistedSequenceNumberConsumer, - String translogUUID + String translogUUID, + TranslogFactory translogFactory ) throws IOException { - return new Translog( + return translogFactory.newTranslog( translogConfig, translogUUID, translogDeletionPolicy, diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogFactory.java b/server/src/main/java/org/opensearch/index/translog/TranslogFactory.java new file mode 100644 index 0000000000000..5500bda99808d --- /dev/null +++ b/server/src/main/java/org/opensearch/index/translog/TranslogFactory.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.translog; + +import java.io.IOException; +import java.util.function.LongConsumer; +import java.util.function.LongSupplier; + +/** + * Translog Factory to enable creation of various local on-disk + * and remote store flavors of {@link Translog} + * + * @opensearch.internal + */ +@FunctionalInterface +public interface TranslogFactory { + + Translog newTranslog( + final TranslogConfig config, + final String translogUUID, + final TranslogDeletionPolicy deletionPolicy, + final LongSupplier globalCheckpointSupplier, + final LongSupplier primaryTermSupplier, + final LongConsumer persistedSequenceNumberConsumer + ) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/index/translog/WriteOnlyTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/WriteOnlyTranslogManager.java index 09f5f38a9f6a9..96a2dd05851c0 100644 --- a/server/src/main/java/org/opensearch/index/translog/WriteOnlyTranslogManager.java +++ b/server/src/main/java/org/opensearch/index/translog/WriteOnlyTranslogManager.java @@ -35,7 +35,8 @@ public WriteOnlyTranslogManager( Supplier localCheckpointTrackerSupplier, String translogUUID, TranslogEventListener translogEventListener, - LifecycleAware engineLifecycleAware + LifecycleAware engineLifecycleAware, + TranslogFactory translogFactory ) throws IOException { super( translogConfig, @@ -47,7 +48,8 @@ public WriteOnlyTranslogManager( localCheckpointTrackerSupplier, translogUUID, translogEventListener, - engineLifecycleAware + engineLifecycleAware, + translogFactory ); } diff --git a/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java b/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java index 7ddd92ea7b36e..269d89352fb18 100644 --- a/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java +++ b/server/src/test/java/org/opensearch/index/engine/EngineConfigFactoryTests.java @@ -16,6 +16,7 @@ import org.opensearch.index.codec.CodecService; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.seqno.RetentionLeases; +import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.TranslogDeletionPolicy; import org.opensearch.index.translog.TranslogDeletionPolicyFactory; import org.opensearch.index.translog.TranslogReader; @@ -66,7 +67,8 @@ public void testCreateEngineConfigFromFactory() { () -> new RetentionLeases(0, 0, Collections.emptyList()), null, null, - false + false, + new InternalTranslogFactory() ); assertNotNull(config.getCodec()); @@ -143,7 +145,8 @@ public void testCreateCodecServiceFromFactory() { () -> new RetentionLeases(0, 0, Collections.emptyList()), null, null, - false + false, + new InternalTranslogFactory() ); assertNotNull(config.getCodec()); } diff --git a/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java b/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java index 1c6d06e9bcc08..1754d6082b86d 100644 --- a/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java +++ b/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java @@ -13,6 +13,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.index.IndexSettings; import org.opensearch.index.seqno.RetentionLeases; +import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.IndexSettingsModule; import org.opensearch.test.OpenSearchTestCase; @@ -102,7 +103,8 @@ private EngineConfig createReadOnlyEngine(IndexSettings indexSettings) { () -> RetentionLeases.EMPTY, null, null, - true + true, + new InternalTranslogFactory() ); } } diff --git a/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java b/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java index 5171f0dfa1d18..234abfba66622 100644 --- a/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java +++ b/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java @@ -47,7 +47,8 @@ public void testRecoveryFromTranslog() throws IOException { () -> tracker, translogUUID, TranslogEventListener.NOOP_TRANSLOG_EVENT_LISTENER, - () -> {} + () -> {}, + new InternalTranslogFactory() ); final int docs = randomIntBetween(1, 100); for (int i = 0; i < docs; i++) { @@ -85,7 +86,8 @@ public void onBeginTranslogRecovery() { beginTranslogRecoveryInvoked.set(true); } }, - () -> {} + () -> {}, + new InternalTranslogFactory() ); AtomicInteger opsRecovered = new AtomicInteger(); int opsRecoveredFromTranslog = translogManager.recoverFromTranslog((snapshot) -> { @@ -122,7 +124,8 @@ public void testTranslogRollsGeneration() throws IOException { () -> tracker, translogUUID, TranslogEventListener.NOOP_TRANSLOG_EVENT_LISTENER, - () -> {} + () -> {}, + new InternalTranslogFactory() ); final int docs = randomIntBetween(1, 100); for (int i = 0; i < docs; i++) { @@ -150,7 +153,8 @@ public void testTranslogRollsGeneration() throws IOException { () -> new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED), translogUUID, TranslogEventListener.NOOP_TRANSLOG_EVENT_LISTENER, - () -> {} + () -> {}, + new InternalTranslogFactory() ); AtomicInteger opsRecovered = new AtomicInteger(); int opsRecoveredFromTranslog = translogManager.recoverFromTranslog((snapshot) -> { @@ -183,7 +187,8 @@ public void testTrimOperationsFromTranslog() throws IOException { () -> tracker, translogUUID, TranslogEventListener.NOOP_TRANSLOG_EVENT_LISTENER, - () -> {} + () -> {}, + new InternalTranslogFactory() ); final int docs = randomIntBetween(1, 100); for (int i = 0; i < docs; i++) { @@ -213,7 +218,8 @@ public void testTrimOperationsFromTranslog() throws IOException { () -> new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED), translogUUID, TranslogEventListener.NOOP_TRANSLOG_EVENT_LISTENER, - () -> {} + () -> {}, + new InternalTranslogFactory() ); AtomicInteger opsRecovered = new AtomicInteger(); int opsRecoveredFromTranslog = translogManager.recoverFromTranslog((snapshot) -> { @@ -260,7 +266,8 @@ public void onAfterTranslogSync() { } } }, - () -> {} + () -> {}, + new InternalTranslogFactory() ); translogManagerAtomicReference.set(translogManager); Engine.Index index = indexForDoc(doc); diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java index 7dedc572ff19b..f446538acccbb 100644 --- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java @@ -90,6 +90,7 @@ import org.opensearch.index.snapshots.IndexShardSnapshotStatus; import org.opensearch.index.store.Store; import org.opensearch.index.store.StoreFileMetadata; +import org.opensearch.index.translog.InternalTranslogFactory; import org.opensearch.index.translog.Translog; import org.opensearch.indices.breaker.CircuitBreakerService; import org.opensearch.indices.breaker.HierarchyCircuitBreakerService; @@ -555,6 +556,7 @@ protected IndexShard newShard( globalCheckpointSyncer, retentionLeaseSyncer, breakerService, + new InternalTranslogFactory(), checkpointPublisher, remoteStore ); From e17af2a19a6fb0258dca5a075cbe273b4367056b Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Tue, 16 Aug 2022 18:06:42 -0400 Subject: [PATCH 17/27] Added bwc version 2.2.1 (#4194) Co-authored-by: opensearch-ci-bot --- .ci/bwcVersions | 1 + server/src/main/java/org/opensearch/Version.java | 1 + 2 files changed, 2 insertions(+) diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 67976d58188e2..1ba3ee562317a 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -48,4 +48,5 @@ BWC_VERSION: - "2.1.0" - "2.1.1" - "2.2.0" + - "2.2.1" - "2.3.0" diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java index 27a39db1c50cb..ba512d3fbcdd9 100644 --- a/server/src/main/java/org/opensearch/Version.java +++ b/server/src/main/java/org/opensearch/Version.java @@ -95,6 +95,7 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_2_1_0 = new Version(2010099, org.apache.lucene.util.Version.LUCENE_9_2_0); public static final Version V_2_1_1 = new Version(2010199, org.apache.lucene.util.Version.LUCENE_9_2_0); public static final Version V_2_2_0 = new Version(2020099, org.apache.lucene.util.Version.LUCENE_9_3_0); + public static final Version V_2_2_1 = new Version(2020199, org.apache.lucene.util.Version.LUCENE_9_3_0); public static final Version V_2_3_0 = new Version(2030099, org.apache.lucene.util.Version.LUCENE_9_3_0); public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_4_0); public static final Version CURRENT = V_3_0_0; From 237f1a5344fb96e888852e089be349867b3c2340 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Tue, 16 Aug 2022 17:28:25 -0700 Subject: [PATCH 18/27] [Segment Replication] Wait for documents to replicate to replica shards (#4236) * [Segment Replication] Add thread sleep to account for replica lag in delete operations test Signed-off-by: Suraj Singh * Address review comments, assertBusy on doc count rather than sleep Signed-off-by: Suraj Singh Signed-off-by: Suraj Singh --- .../replication/SegmentReplicationIT.java | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 96ac703b9837e..9518d5e802a8e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; @@ -293,9 +294,22 @@ public void testDeleteOperations() throws Exception { refresh(INDEX_NAME); waitForReplicaUpdate(); - - assertHitCount(client(nodeA).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount - 1); - assertHitCount(client(nodeB).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedHitCount - 1); + assertBusy(() -> { + final long nodeA_Count = client(nodeA).prepareSearch(INDEX_NAME) + .setSize(0) + .setPreference("_only_local") + .get() + .getHits() + .getTotalHits().value; + assertEquals(expectedHitCount - 1, nodeA_Count); + final long nodeB_Count = client(nodeB).prepareSearch(INDEX_NAME) + .setSize(0) + .setPreference("_only_local") + .get() + .getHits() + .getTotalHits().value; + assertEquals(expectedHitCount - 1, nodeB_Count); + }, 5, TimeUnit.SECONDS); } } From 3a97d4cad65fa19126340aabf7e8bd2b1ce2b822 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Tue, 16 Aug 2022 18:39:51 -0700 Subject: [PATCH 19/27] Segment Replication - Add additional unit tests for update & delete ops. (#4237) * Segment Replication - Add additional unit tests for update & delete operations. Signed-off-by: Marc Handalian * Fix spotless. Signed-off-by: Marc Handalian Signed-off-by: Marc Handalian --- .../SegmentReplicationIndexShardTests.java | 54 +++++++++++++++++++ ...enSearchIndexLevelReplicationTestCase.java | 7 ++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index d10f8ced963b7..4f6d86c13e12c 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -8,17 +8,23 @@ package org.opensearch.index.shard; +import org.opensearch.action.delete.DeleteRequest; +import org.opensearch.action.index.IndexRequest; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexSettings; +import org.opensearch.index.engine.DocIdSeqNoAndSource; import org.opensearch.index.engine.NRTReplicationEngineFactory; +import org.opensearch.index.mapper.MapperService; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher; import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.replication.common.ReplicationType; import java.io.IOException; +import java.util.List; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.verify; @@ -52,6 +58,54 @@ public void testReplicationCheckpointNotNullForSegReb() throws IOException { closeShards(indexShard); } + public void testSegmentReplication_Index_Update_Delete() throws Exception { + String mappings = "{ \"" + MapperService.SINGLE_MAPPING_NAME + "\": { \"properties\": { \"foo\": { \"type\": \"keyword\"} }}}"; + try (ReplicationGroup shards = createGroup(2, settings, mappings, new NRTReplicationEngineFactory())) { + shards.startAll(); + final IndexShard primaryShard = shards.getPrimary(); + + final int numDocs = randomIntBetween(100, 200); + for (int i = 0; i < numDocs; i++) { + shards.index(new IndexRequest(index.getName()).id(String.valueOf(i)).source("{\"foo\": \"bar\"}", XContentType.JSON)); + } + + primaryShard.refresh("Test"); + replicateSegments(primaryShard, shards.getReplicas()); + + shards.assertAllEqual(numDocs); + + for (int i = 0; i < numDocs; i++) { + // randomly update docs. + if (randomBoolean()) { + shards.index( + new IndexRequest(index.getName()).id(String.valueOf(i)).source("{ \"foo\" : \"baz\" }", XContentType.JSON) + ); + } + } + + primaryShard.refresh("Test"); + replicateSegments(primaryShard, shards.getReplicas()); + shards.assertAllEqual(numDocs); + + final List docs = getDocIdAndSeqNos(primaryShard); + for (IndexShard shard : shards.getReplicas()) { + assertEquals(getDocIdAndSeqNos(shard), docs); + } + for (int i = 0; i < numDocs; i++) { + // randomly delete. + if (randomBoolean()) { + shards.delete(new DeleteRequest(index.getName()).id(String.valueOf(i))); + } + } + primaryShard.refresh("Test"); + replicateSegments(primaryShard, shards.getReplicas()); + final List docsAfterDelete = getDocIdAndSeqNos(primaryShard); + for (IndexShard shard : shards.getReplicas()) { + assertEquals(getDocIdAndSeqNos(shard), docsAfterDelete); + } + } + } + public void testIgnoreShardIdle() throws Exception { try (ReplicationGroup shards = createGroup(1, settings, new NRTReplicationEngineFactory())) { shards.startAll(); diff --git a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java index 249ffcfd0bf6e..b3f062aef4fbe 100644 --- a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java @@ -141,7 +141,12 @@ protected ReplicationGroup createGroup(int replicas, Settings settings) throws I } protected ReplicationGroup createGroup(int replicas, Settings settings, EngineFactory engineFactory) throws IOException { - IndexMetadata metadata = buildIndexMetadata(replicas, settings, indexMapping); + return createGroup(replicas, settings, indexMapping, engineFactory); + } + + protected ReplicationGroup createGroup(int replicas, Settings settings, String mappings, EngineFactory engineFactory) + throws IOException { + IndexMetadata metadata = buildIndexMetadata(replicas, settings, mappings); return new ReplicationGroup(metadata) { @Override protected EngineFactory getEngineFactory(ShardRouting routing) { From f65e02d1b910bd0a1990868bfa5d12ba829bbbd5 Mon Sep 17 00:00:00 2001 From: Marc Handalian Date: Wed, 17 Aug 2022 10:32:11 -0700 Subject: [PATCH 20/27] Support shard promotion with Segment Replication. (#4135) * Support shard promotion with Segment Replication. This change adds basic failover support with segment replication. Once selected, a replica will commit and reopen a writeable engine. Signed-off-by: Marc Handalian * Add check to ensure a closed shard does not publish checkpoints. Signed-off-by: Marc Handalian * Clean up in SegmentReplicationIT. Signed-off-by: Marc Handalian * PR feedback. Signed-off-by: Marc Handalian * Fix merge conflict. Signed-off-by: Marc Handalian Signed-off-by: Marc Handalian --- .../replication/SegmentReplicationIT.java | 148 +++++++++++++++++- .../org/opensearch/index/IndexService.java | 2 +- .../index/engine/NRTReplicationEngine.java | 17 ++ .../shard/CheckpointRefreshListener.java | 2 +- .../opensearch/index/shard/IndexShard.java | 40 +++-- .../org/opensearch/index/store/Store.java | 42 +++++ .../engine/NRTReplicationEngineTests.java | 52 ++++++ .../SegmentReplicationIndexShardTests.java | 80 +++++++++- 8 files changed, 363 insertions(+), 20 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java index 9518d5e802a8e..8566cc5556861 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationIT.java @@ -19,7 +19,10 @@ import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.node.DiscoveryNode; +import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.allocation.command.CancelAllocationCommand; +import org.opensearch.common.Nullable; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.Index; @@ -30,6 +33,7 @@ import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.BackgroundIndexer; +import org.opensearch.test.InternalTestCluster; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; @@ -73,6 +77,109 @@ protected boolean addMockInternalEngine() { return false; } + public void testPrimaryStopped_ReplicaPromoted() throws Exception { + final String primary = internalCluster().startNode(); + createIndex(INDEX_NAME); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final String replica = internalCluster().startNode(); + ensureGreen(INDEX_NAME); + + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + refresh(INDEX_NAME); + + waitForReplicaUpdate(); + assertHitCount(client(primary).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 1); + assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 1); + + // index another doc but don't refresh, we will ensure this is searchable once replica is promoted. + client().prepareIndex(INDEX_NAME).setId("2").setSource("bar", "baz").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + + // stop the primary node - we only have one shard on here. + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary)); + ensureYellowAndNoInitializingShards(INDEX_NAME); + + final ShardRouting replicaShardRouting = getShardRoutingForNodeName(replica); + assertNotNull(replicaShardRouting); + assertTrue(replicaShardRouting + " should be promoted as a primary", replicaShardRouting.primary()); + assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 2); + + // assert we can index into the new primary. + client().prepareIndex(INDEX_NAME).setId("3").setSource("bar", "baz").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 3); + + // start another node, index another doc and replicate. + String nodeC = internalCluster().startNode(); + ensureGreen(INDEX_NAME); + client().prepareIndex(INDEX_NAME).setId("4").setSource("baz", "baz").get(); + refresh(INDEX_NAME); + waitForReplicaUpdate(); + assertHitCount(client(nodeC).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 4); + assertHitCount(client(replica).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 4); + assertSegmentStats(REPLICA_COUNT); + } + + public void testRestartPrimary() throws Exception { + final String primary = internalCluster().startNode(); + createIndex(INDEX_NAME); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final String replica = internalCluster().startNode(); + ensureGreen(INDEX_NAME); + + assertEquals(getNodeContainingPrimaryShard().getName(), primary); + + final int initialDocCount = 1; + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + refresh(INDEX_NAME); + + waitForReplicaUpdate(); + assertDocCounts(initialDocCount, replica, primary); + + internalCluster().restartNode(primary); + ensureGreen(INDEX_NAME); + + assertEquals(getNodeContainingPrimaryShard().getName(), replica); + + flushAndRefresh(INDEX_NAME); + waitForReplicaUpdate(); + + assertDocCounts(initialDocCount, replica, primary); + assertSegmentStats(REPLICA_COUNT); + } + + public void testCancelPrimaryAllocation() throws Exception { + // this test cancels allocation on the primary - promoting the new replica and recreating the former primary as a replica. + final String primary = internalCluster().startNode(); + createIndex(INDEX_NAME); + ensureYellowAndNoInitializingShards(INDEX_NAME); + final String replica = internalCluster().startNode(); + ensureGreen(INDEX_NAME); + + final int initialDocCount = 1; + + client().prepareIndex(INDEX_NAME).setId("1").setSource("foo", "bar").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get(); + refresh(INDEX_NAME); + + waitForReplicaUpdate(); + assertDocCounts(initialDocCount, replica, primary); + + final IndexShard indexShard = getIndexShard(primary); + client().admin() + .cluster() + .prepareReroute() + .add(new CancelAllocationCommand(INDEX_NAME, indexShard.shardId().id(), primary, true)) + .execute() + .actionGet(); + ensureGreen(INDEX_NAME); + + assertEquals(getNodeContainingPrimaryShard().getName(), replica); + + flushAndRefresh(INDEX_NAME); + waitForReplicaUpdate(); + + assertDocCounts(initialDocCount, replica, primary); + assertSegmentStats(REPLICA_COUNT); + } + public void testReplicationAfterPrimaryRefreshAndFlush() throws Exception { final String nodeA = internalCluster().startNode(); final String nodeB = internalCluster().startNode(); @@ -240,9 +347,8 @@ public void testStartReplicaAfterPrimaryIndexesDocs() throws Exception { assertHitCount(client(primaryNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 3); assertHitCount(client(replicaNode).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), 3); - final Index index = resolveIndex(INDEX_NAME); - IndexShard primaryShard = getIndexShard(index, primaryNode); - IndexShard replicaShard = getIndexShard(index, replicaNode); + IndexShard primaryShard = getIndexShard(primaryNode); + IndexShard replicaShard = getIndexShard(replicaNode); assertEquals( primaryShard.translogStats().estimatedNumberOfOperations(), replicaShard.translogStats().estimatedNumberOfOperations() @@ -351,8 +457,7 @@ private void assertSegmentStats(int numberOfReplicas) throws IOException { final ShardRouting replicaShardRouting = shardSegment.getShardRouting(); ClusterState state = client(internalCluster().getMasterName()).admin().cluster().prepareState().get().getState(); final DiscoveryNode replicaNode = state.nodes().resolveNode(replicaShardRouting.currentNodeId()); - final Index index = resolveIndex(INDEX_NAME); - IndexShard indexShard = getIndexShard(index, replicaNode.getName()); + IndexShard indexShard = getIndexShard(replicaNode.getName()); final String lastCommitSegmentsFileName = SegmentInfos.getLastCommitSegmentsFileName(indexShard.store().directory()); // calls to readCommit will fail if a valid commit point and all its segments are not in the store. SegmentInfos.readCommit(indexShard.store().directory(), lastCommitSegmentsFileName); @@ -392,7 +497,8 @@ private void waitForReplicaUpdate() throws Exception { }); } - private IndexShard getIndexShard(Index index, String node) { + private IndexShard getIndexShard(String node) { + final Index index = resolveIndex(INDEX_NAME); IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node); IndexService indexService = indicesService.indexServiceSafe(index); final Optional shardId = indexService.shardIds().stream().findFirst(); @@ -409,7 +515,8 @@ private List getShardSegments(IndicesSegmentResponse indicesSeg } private Map getLatestSegments(ShardSegments segments) { - final Long latestPrimaryGen = segments.getSegments().stream().map(Segment::getGeneration).max(Long::compare).get(); + final Optional generation = segments.getSegments().stream().map(Segment::getGeneration).max(Long::compare); + final Long latestPrimaryGen = generation.get(); return segments.getSegments() .stream() .filter(s -> s.getGeneration() == latestPrimaryGen) @@ -419,4 +526,31 @@ private Map getLatestSegments(ShardSegments segments) { private Map> segmentsByShardType(ShardSegments[] replicationGroupSegments) { return Arrays.stream(replicationGroupSegments).collect(Collectors.groupingBy(s -> s.getShardRouting().primary())); } + + @Nullable + private ShardRouting getShardRoutingForNodeName(String nodeName) { + final ClusterState state = client(internalCluster().getClusterManagerName()).admin().cluster().prepareState().get().getState(); + for (IndexShardRoutingTable shardRoutingTable : state.routingTable().index(INDEX_NAME)) { + for (ShardRouting shardRouting : shardRoutingTable.activeShards()) { + final String nodeId = shardRouting.currentNodeId(); + final DiscoveryNode discoveryNode = state.nodes().resolveNode(nodeId); + if (discoveryNode.getName().equals(nodeName)) { + return shardRouting; + } + } + } + return null; + } + + private void assertDocCounts(int expectedDocCount, String... nodeNames) { + for (String node : nodeNames) { + assertHitCount(client(node).prepareSearch(INDEX_NAME).setSize(0).setPreference("_only_local").get(), expectedDocCount); + } + } + + private DiscoveryNode getNodeContainingPrimaryShard() { + final ClusterState state = client(internalCluster().getClusterManagerName()).admin().cluster().prepareState().get().getState(); + final ShardRouting primaryShard = state.routingTable().index(INDEX_NAME).shard(0).primaryShard(); + return state.nodes().resolveNode(primaryShard.currentNodeId()); + } } diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java index 38065bf5aed20..e1427df1c34ab 100644 --- a/server/src/main/java/org/opensearch/index/IndexService.java +++ b/server/src/main/java/org/opensearch/index/IndexService.java @@ -550,7 +550,7 @@ public synchronized IndexShard createShard( circuitBreakerService, // TODO Replace with remote translog factory in the follow up PR this.indexSettings.isRemoteTranslogStoreEnabled() ? null : new InternalTranslogFactory(), - this.indexSettings.isSegRepEnabled() && routing.primary() ? checkpointPublisher : null, + this.indexSettings.isSegRepEnabled() ? checkpointPublisher : null, remoteStore ); eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created"); diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java index a481203ba3dea..6f5b7030ed65f 100644 --- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java +++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java @@ -129,6 +129,23 @@ public synchronized void updateSegments(final SegmentInfos infos, long seqNo) th localCheckpointTracker.fastForwardProcessedSeqNo(seqNo); } + /** + * Persist the latest live SegmentInfos. + * + * This method creates a commit point from the latest SegmentInfos. It is intended to be used when this shard is about to be promoted as the new primary. + * + * TODO: If this method is invoked while the engine is currently updating segments on its reader, wait for that update to complete so the updated segments are used. + * + * + * @throws IOException - When there is an IO error committing the SegmentInfos. + */ + public void commitSegmentInfos() throws IOException { + // TODO: This method should wait for replication events to finalize. + final SegmentInfos latestSegmentInfos = getLatestSegmentInfos(); + store.commitSegmentInfos(latestSegmentInfos, localCheckpointTracker.getMaxSeqNo(), localCheckpointTracker.getProcessedCheckpoint()); + translogManager.syncTranslog(); + } + @Override public String getHistoryUUID() { return loadHistoryUUID(lastCommittedSegmentInfos.userData); diff --git a/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java index 96d74bea85920..fb046e2310d93 100644 --- a/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java +++ b/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java @@ -40,7 +40,7 @@ public void beforeRefresh() throws IOException { @Override public void afterRefresh(boolean didRefresh) throws IOException { - if (didRefresh && shard.getReplicationTracker().isPrimaryMode()) { + if (didRefresh && shard.state() != IndexShardState.CLOSED && shard.getReplicationTracker().isPrimaryMode()) { publisher.publish(shard); } } diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 8970e0734f184..67a8e691fda0d 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -242,6 +242,7 @@ public class IndexShard extends AbstractIndexShardComponent implements IndicesCl private final GlobalCheckpointListeners globalCheckpointListeners; private final PendingReplicationActions pendingReplicationActions; private final ReplicationTracker replicationTracker; + private final SegmentReplicationCheckpointPublisher checkpointPublisher; protected volatile ShardRouting shardRouting; protected volatile IndexShardState state; @@ -306,8 +307,6 @@ Runnable getGlobalCheckpointSyncer() { private final AtomicReference pendingRefreshLocation = new AtomicReference<>(); private final RefreshPendingLocationListener refreshPendingLocationListener; private volatile boolean useRetentionLeasesInPeerRecovery; - private final ReferenceManager.RefreshListener checkpointRefreshListener; - private final Store remoteStore; private final TranslogFactory translogFactory; @@ -417,11 +416,7 @@ public boolean shouldCache(Query query) { persistMetadata(path, indexSettings, shardRouting, null, logger); this.useRetentionLeasesInPeerRecovery = replicationTracker.hasAllPeerRecoveryRetentionLeases(); this.refreshPendingLocationListener = new RefreshPendingLocationListener(); - if (checkpointPublisher != null) { - this.checkpointRefreshListener = new CheckpointRefreshListener(this, checkpointPublisher); - } else { - this.checkpointRefreshListener = null; - } + this.checkpointPublisher = checkpointPublisher; this.remoteStore = remoteStore; this.translogFactory = translogFactory; } @@ -627,6 +622,11 @@ public void updateShardState( + newRouting; assert getOperationPrimaryTerm() == newPrimaryTerm; try { + if (indexSettings.isSegRepEnabled()) { + // this Shard's engine was read only, we need to update its engine before restoring local history from xlog. + assert newRouting.primary() && currentRouting.primary() == false; + promoteNRTReplicaToPrimary(); + } replicationTracker.activatePrimaryMode(getLocalCheckpoint()); ensurePeerRecoveryRetentionLeasesExist(); /* @@ -3231,8 +3231,8 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) thro Directory remoteDirectory = ((FilterDirectory) ((FilterDirectory) remoteStore.directory()).getDelegate()).getDelegate(); internalRefreshListener.add(new RemoteStoreRefreshListener(store.directory(), remoteDirectory)); } - if (this.checkpointRefreshListener != null) { - internalRefreshListener.add(checkpointRefreshListener); + if (this.checkpointPublisher != null && indexSettings.isSegRepEnabled() && shardRouting.primary()) { + internalRefreshListener.add(new CheckpointRefreshListener(this, this.checkpointPublisher)); } return this.engineConfigFactory.newEngineConfig( @@ -4123,4 +4123,26 @@ RetentionLeaseSyncer getRetentionLeaseSyncer() { public GatedCloseable getSegmentInfosSnapshot() { return getEngine().getSegmentInfosSnapshot(); } + + /** + * With segment replication enabled - prepare the shard's engine to be promoted as the new primary. + * + * If this shard is currently using a replication engine, this method: + * 1. Invokes {@link NRTReplicationEngine#commitSegmentInfos()} to ensure the engine can be reopened as writeable from the latest refresh point. + * InternalEngine opens its IndexWriter from an on-disk commit point, but this replica may have recently synced from a primary's refresh point, meaning it has documents searchable in its in-memory SegmentInfos + * that are not part of a commit point. This ensures that those documents are made part of a commit and do not need to be reindexed after promotion. + * 2. Invokes resetEngineToGlobalCheckpoint - This call performs the engine swap, opening up as a writeable engine and replays any operations in the xlog. The operations indexed from xlog here will be + * any ack'd writes that were not copied to this replica before promotion. + */ + private void promoteNRTReplicaToPrimary() { + assert shardRouting.primary() && indexSettings.isSegRepEnabled(); + getReplicationEngine().ifPresentOrElse(engine -> { + try { + engine.commitSegmentInfos(); + resetEngineToGlobalCheckpoint(); + } catch (IOException e) { + throw new EngineException(shardId, "Unable to update replica to writeable engine, failing shard", e); + } + }, () -> { throw new EngineException(shardId, "Expected replica engine to be of type NRTReplicationEngine"); }); + } } diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java index 163717ad94c2c..58598ab2d08f4 100644 --- a/server/src/main/java/org/opensearch/index/store/Store.java +++ b/server/src/main/java/org/opensearch/index/store/Store.java @@ -121,6 +121,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; +import static org.opensearch.index.seqno.SequenceNumbers.LOCAL_CHECKPOINT_KEY; /** * A Store provides plain access to files written by an opensearch index shard. Each shard @@ -799,6 +800,47 @@ public void beforeClose() { shardLock.setDetails("closing shard"); } + /** + * This method should only be used with Segment Replication. + * Perform a commit from a live {@link SegmentInfos}. Replica engines with segrep do not have an IndexWriter and Lucene does not currently + * have the ability to create a writer directly from a SegmentInfos object. To promote the replica as a primary and avoid reindexing, we must first commit + * on the replica so that it can be opened with a writeable engine. Further, InternalEngine currently invokes `trimUnsafeCommits` which reverts the engine to a previous safeCommit where the max seqNo is less than or equal + * to the current global checkpoint. It is likely that the replica has a maxSeqNo that is higher than the global cp and a new commit will be wiped. + * + * To get around these limitations, this method first creates an IndexCommit directly from SegmentInfos, it then + * uses an appending IW to create an IndexCommit from the commit created on SegmentInfos. + * This ensures that 1. All files in the new commit are fsynced and 2. Deletes older commit points so the only commit to start from is our new commit. + * + * @param latestSegmentInfos {@link SegmentInfos} The latest active infos + * @param maxSeqNo The engine's current maxSeqNo + * @param processedCheckpoint The engine's current processed checkpoint. + * @throws IOException when there is an IO error committing. + */ + public void commitSegmentInfos(SegmentInfos latestSegmentInfos, long maxSeqNo, long processedCheckpoint) throws IOException { + assert indexSettings.isSegRepEnabled(); + metadataLock.writeLock().lock(); + try { + final Map userData = new HashMap<>(latestSegmentInfos.getUserData()); + userData.put(LOCAL_CHECKPOINT_KEY, String.valueOf(processedCheckpoint)); + userData.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(maxSeqNo)); + latestSegmentInfos.setUserData(userData, true); + latestSegmentInfos.commit(directory()); + + // similar to TrimUnsafeCommits, create a commit with an appending IW, this will delete old commits and ensure all files + // associated with the SegmentInfos.commit are fsynced. + final List existingCommits = DirectoryReader.listCommits(directory); + assert existingCommits.isEmpty() == false : "Expected at least one commit but none found"; + final IndexCommit lastIndexCommit = existingCommits.get(existingCommits.size() - 1); + assert latestSegmentInfos.getSegmentsFileName().equals(lastIndexCommit.getSegmentsFileName()); + try (IndexWriter writer = newAppendingIndexWriter(directory, lastIndexCommit)) { + writer.setLiveCommitData(lastIndexCommit.getUserData().entrySet()); + writer.commit(); + } + } finally { + metadataLock.writeLock().unlock(); + } + } + /** * A store directory * diff --git a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java index 675ff860c3334..1fe1a37dedae0 100644 --- a/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java +++ b/server/src/test/java/org/opensearch/index/engine/NRTReplicationEngineTests.java @@ -12,18 +12,25 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.SegmentInfos; import org.hamcrest.MatcherAssert; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.concurrent.GatedCloseable; import org.opensearch.common.lucene.Lucene; import org.opensearch.common.lucene.search.Queries; +import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexSettings; import org.opensearch.index.mapper.ParsedDocument; +import org.opensearch.index.seqno.LocalCheckpointTracker; import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.store.Store; import org.opensearch.index.translog.TestTranslog; import org.opensearch.index.translog.Translog; +import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.test.IndexSettingsModule; import java.io.IOException; import java.nio.file.Path; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -31,6 +38,8 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.opensearch.index.seqno.SequenceNumbers.NO_OPS_PERFORMED; +import static org.opensearch.index.seqno.SequenceNumbers.LOCAL_CHECKPOINT_KEY; +import static org.opensearch.index.seqno.SequenceNumbers.MAX_SEQ_NO; public class NRTReplicationEngineTests extends EngineTestCase { @@ -210,6 +219,49 @@ public void testTrimTranslogOps() throws Exception { } } + public void testCommitSegmentInfos() throws Exception { + // This test asserts that NRTReplication#commitSegmentInfos creates a new commit point with the latest checkpoints + // stored in user data. + final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); + final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( + "index", + Settings.builder().put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT).build() + ); + try ( + final Store nrtEngineStore = createStore(indexSettings, newDirectory()); + final NRTReplicationEngine nrtEngine = buildNrtReplicaEngine(globalCheckpoint, nrtEngineStore) + ) { + List operations = generateHistoryOnReplica(between(1, 500), randomBoolean(), randomBoolean(), randomBoolean()) + .stream() + .filter(op -> op.operationType().equals(Engine.Operation.TYPE.INDEX)) + .collect(Collectors.toList()); + for (Engine.Operation op : operations) { + applyOperation(nrtEngine, op); + } + + final SegmentInfos previousInfos = nrtEngine.getLatestSegmentInfos(); + LocalCheckpointTracker localCheckpointTracker = nrtEngine.getLocalCheckpointTracker(); + final long maxSeqNo = localCheckpointTracker.getMaxSeqNo(); + final long processedCheckpoint = localCheckpointTracker.getProcessedCheckpoint(); + nrtEngine.commitSegmentInfos(); + + // ensure getLatestSegmentInfos returns an updated infos ref with correct userdata. + final SegmentInfos latestSegmentInfos = nrtEngine.getLatestSegmentInfos(); + assertEquals(previousInfos.getGeneration(), latestSegmentInfos.getLastGeneration()); + Map userData = latestSegmentInfos.getUserData(); + assertEquals(processedCheckpoint, localCheckpointTracker.getProcessedCheckpoint()); + assertEquals(maxSeqNo, Long.parseLong(userData.get(MAX_SEQ_NO))); + assertEquals(processedCheckpoint, Long.parseLong(userData.get(LOCAL_CHECKPOINT_KEY))); + + // read infos from store and assert userdata + final String lastCommitSegmentsFileName = SegmentInfos.getLastCommitSegmentsFileName(nrtEngineStore.directory()); + final SegmentInfos committedInfos = SegmentInfos.readCommit(nrtEngineStore.directory(), lastCommitSegmentsFileName); + userData = committedInfos.getUserData(); + assertEquals(processedCheckpoint, Long.parseLong(userData.get(LOCAL_CHECKPOINT_KEY))); + assertEquals(maxSeqNo, Long.parseLong(userData.get(MAX_SEQ_NO))); + } + } + private void assertMatchingSegmentsAndCheckpoints(NRTReplicationEngine nrtEngine, SegmentInfos expectedSegmentInfos) throws IOException { assertEquals(engine.getPersistedLocalCheckpoint(), nrtEngine.getPersistedLocalCheckpoint()); diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java index 4f6d86c13e12c..23371a39871c7 100644 --- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java +++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java @@ -16,6 +16,8 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.index.IndexSettings; import org.opensearch.index.engine.DocIdSeqNoAndSource; +import org.opensearch.index.engine.InternalEngine; +import org.opensearch.index.engine.NRTReplicationEngine; import org.opensearch.index.engine.NRTReplicationEngineFactory; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.replication.OpenSearchIndexLevelReplicationTestCase; @@ -26,10 +28,12 @@ import java.io.IOException; import java.util.List; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; +import static java.util.Arrays.asList; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; public class SegmentReplicationIndexShardTests extends OpenSearchIndexLevelReplicationTestCase { @@ -177,4 +181,76 @@ public void testPublishCheckpointAfterRelocationHandOff() throws IOException { closeShards(shard); } + public void testNRTReplicaPromotedAsPrimary() throws Exception { + try (ReplicationGroup shards = createGroup(2, settings, new NRTReplicationEngineFactory())) { + shards.startAll(); + IndexShard oldPrimary = shards.getPrimary(); + final IndexShard nextPrimary = shards.getReplicas().get(0); + final IndexShard replica = shards.getReplicas().get(1); + + // 1. Create ops that are in the index and xlog of both shards but not yet part of a commit point. + final int numDocs = shards.indexDocs(randomInt(10)); + + // refresh and copy the segments over. + oldPrimary.refresh("Test"); + replicateSegments(oldPrimary, shards.getReplicas()); + + // at this point both shards should have numDocs persisted and searchable. + assertDocCounts(oldPrimary, numDocs, numDocs); + for (IndexShard shard : shards.getReplicas()) { + assertDocCounts(shard, numDocs, numDocs); + } + + // 2. Create ops that are in the replica's xlog, not in the index. + // index some more into both but don't replicate. replica will have only numDocs searchable, but should have totalDocs + // persisted. + final int totalDocs = numDocs + shards.indexDocs(randomInt(10)); + + assertDocCounts(oldPrimary, totalDocs, totalDocs); + for (IndexShard shard : shards.getReplicas()) { + assertDocCounts(shard, totalDocs, numDocs); + } + + // promote the replica + shards.syncGlobalCheckpoint(); + assertEquals(totalDocs, nextPrimary.translogStats().estimatedNumberOfOperations()); + shards.promoteReplicaToPrimary(nextPrimary); + + // close and start the oldPrimary as a replica. + oldPrimary.close("demoted", false); + oldPrimary.store().close(); + oldPrimary = shards.addReplicaWithExistingPath(oldPrimary.shardPath(), oldPrimary.routingEntry().currentNodeId()); + shards.recoverReplica(oldPrimary); + + assertEquals(NRTReplicationEngine.class, oldPrimary.getEngine().getClass()); + assertEquals(InternalEngine.class, nextPrimary.getEngine().getClass()); + assertDocCounts(nextPrimary, totalDocs, totalDocs); + assertEquals(0, nextPrimary.translogStats().estimatedNumberOfOperations()); + + // refresh and push segments to our other replica. + nextPrimary.refresh("test"); + replicateSegments(nextPrimary, asList(replica)); + + for (IndexShard shard : shards) { + assertConsistentHistoryBetweenTranslogAndLucene(shard); + } + final List docsAfterRecovery = getDocIdAndSeqNos(shards.getPrimary()); + for (IndexShard shard : shards.getReplicas()) { + assertThat(shard.routingEntry().toString(), getDocIdAndSeqNos(shard), equalTo(docsAfterRecovery)); + } + } + } + + /** + * Assert persisted and searchable doc counts. This method should not be used while docs are concurrently indexed because + * it asserts point in time seqNos are relative to the doc counts. + */ + private void assertDocCounts(IndexShard indexShard, int expectedPersistedDocCount, int expectedSearchableDocCount) throws IOException { + assertDocCount(indexShard, expectedSearchableDocCount); + // assigned seqNos start at 0, so assert max & local seqNos are 1 less than our persisted doc count. + assertEquals(expectedPersistedDocCount - 1, indexShard.seqNoStats().getMaxSeqNo()); + assertEquals(expectedPersistedDocCount - 1, indexShard.seqNoStats().getLocalCheckpoint()); + // processed cp should be 1 less than our searchable doc count. + assertEquals(expectedSearchableDocCount - 1, indexShard.getProcessedLocalCheckpoint()); + } } From a2ba3a8c6662b0bac5fc3d73c5029fe323f1192b Mon Sep 17 00:00:00 2001 From: Kartik Ganesh Date: Wed, 17 Aug 2022 14:27:07 -0700 Subject: [PATCH 21/27] Added timing data and more granular stages to SegmentReplicationState (#4222) * Added timing data and more granular stages to SegmentReplicationState This change introduces instrumentation logging that measures the latency of the various stages of segment replication as seen by each replica. Logs have also been added to the source node for checkpoint publishing and checkpoint metadata responses. All logging is currently at the TRACE level. Signed-off-by: Kartik Ganesh * Fixing SegmentReplicationTarget tests Signed-off-by: Kartik Ganesh * Incorporated PR feedback Signed-off-by: Kartik Ganesh * Fixing SegmentReplicationTargetService tests Signed-off-by: Kartik Ganesh Signed-off-by: Kartik Ganesh --- .../SegmentReplicationSourceHandler.java | 18 ++++- .../SegmentReplicationSourceService.java | 14 ++++ .../replication/SegmentReplicationState.java | 71 +++++++++++++++---- .../replication/SegmentReplicationTarget.java | 19 +++-- .../SegmentReplicationTargetService.java | 22 +++++- .../checkpoint/PublishCheckpointAction.java | 24 ++++++- .../SegmentReplicationTargetServiceTests.java | 21 ++++-- .../SegmentReplicationTargetTests.java | 20 ++++-- 8 files changed, 175 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java index ce764900e433f..2d21653c1924c 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java @@ -27,6 +27,7 @@ import org.opensearch.indices.recovery.FileChunkWriter; import org.opensearch.indices.recovery.MultiChunkTransfer; import org.opensearch.indices.replication.common.CopyState; +import org.opensearch.indices.replication.common.ReplicationTimer; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.Transports; @@ -104,16 +105,24 @@ class SegmentReplicationSourceHandler { * @param listener {@link ActionListener} that completes with the list of files sent. */ public synchronized void sendFiles(GetSegmentFilesRequest request, ActionListener listener) { + final ReplicationTimer timer = new ReplicationTimer(); if (isReplicating.compareAndSet(false, true) == false) { throw new OpenSearchException("Replication to {} is already running.", shard.shardId()); } future.addListener(listener, OpenSearchExecutors.newDirectExecutorService()); final Closeable releaseResources = () -> IOUtils.close(resources); try { - + timer.start(); final Consumer onFailure = e -> { assert Transports.assertNotTransportThread(SegmentReplicationSourceHandler.this + "[onFailure]"); IOUtils.closeWhileHandlingException(releaseResources, () -> future.onFailure(e)); + timer.stop(); + logger.trace( + "[replication id {}] Source node failed to send files to target node [{}], timing: {}", + request.getReplicationId(), + request.getTargetNode().getId(), + timer.time() + ); }; RunUnderPrimaryPermit.run(() -> { @@ -151,6 +160,13 @@ public synchronized void sendFiles(GetSegmentFilesRequest request, ActionListene future.onResponse(new GetSegmentFilesResponse(List.of(storeFileMetadata))); } finally { IOUtils.close(resources); + timer.stop(); + logger.trace( + "[replication id {}] Source node completed sending files to target node [{}], timing: {}", + request.getReplicationId(), + request.getTargetNode().getId(), + timer.time() + ); } }, onFailure); } catch (Exception e) { diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java index d428459884f97..0cee731fde2cb 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.action.support.ChannelActionListener; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterStateListener; @@ -25,6 +26,7 @@ import org.opensearch.indices.recovery.RecoverySettings; import org.opensearch.indices.recovery.RetryableTransportClient; import org.opensearch.indices.replication.common.CopyState; +import org.opensearch.indices.replication.common.ReplicationTimer; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportChannel; @@ -86,6 +88,8 @@ public SegmentReplicationSourceService( private class CheckpointInfoRequestHandler implements TransportRequestHandler { @Override public void messageReceived(CheckpointInfoRequest request, TransportChannel channel, Task task) throws Exception { + final ReplicationTimer timer = new ReplicationTimer(); + timer.start(); final RemoteSegmentFileChunkWriter segmentSegmentFileChunkWriter = new RemoteSegmentFileChunkWriter( request.getReplicationId(), recoverySettings, @@ -109,6 +113,16 @@ public void messageReceived(CheckpointInfoRequest request, TransportChannel chan copyState.getPendingDeleteFiles() ) ); + timer.stop(); + logger.trace( + new ParameterizedMessage( + "[replication id {}] Source node sent checkpoint info [{}] to target node [{}], timing: {}", + request.getReplicationId(), + copyState.getCheckpoint(), + request.getTargetNode().getId(), + timer.time() + ) + ); } } diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java index 838c06a4785ef..f865ba1332186 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationState.java @@ -8,10 +8,14 @@ package org.opensearch.indices.replication; +import org.opensearch.common.collect.Tuple; import org.opensearch.indices.replication.common.ReplicationLuceneIndex; import org.opensearch.indices.replication.common.ReplicationState; import org.opensearch.indices.replication.common.ReplicationTimer; +import java.util.ArrayList; +import java.util.List; + /** * ReplicationState implementation to track Segment Replication events. * @@ -26,10 +30,12 @@ public class SegmentReplicationState implements ReplicationState { */ public enum Stage { DONE((byte) 0), - INIT((byte) 1), - - REPLICATING((byte) 2); + REPLICATING((byte) 2), + GET_CHECKPOINT_INFO((byte) 3), + FILE_DIFF((byte) 4), + GET_FILES((byte) 5), + FINALIZE_REPLICATION((byte) 6); private static final Stage[] STAGES = new Stage[Stage.values().length]; @@ -60,13 +66,27 @@ public static Stage fromId(byte id) { private Stage stage; private final ReplicationLuceneIndex index; - private final ReplicationTimer timer; + private final ReplicationTimer overallTimer; + private final ReplicationTimer stageTimer; + private final List> timingData; + private long replicationId; public SegmentReplicationState(ReplicationLuceneIndex index) { stage = Stage.INIT; this.index = index; - timer = new ReplicationTimer(); - timer.start(); + // Timing data will have as many entries as stages, plus one + // additional entry for the overall timer + timingData = new ArrayList<>(Stage.values().length + 1); + overallTimer = new ReplicationTimer(); + stageTimer = new ReplicationTimer(); + stageTimer.start(); + // set an invalid value by default + this.replicationId = -1L; + } + + public SegmentReplicationState(ReplicationLuceneIndex index, long replicationId) { + this(index); + this.replicationId = replicationId; } @Override @@ -74,9 +94,17 @@ public ReplicationLuceneIndex getIndex() { return index; } + public long getReplicationId() { + return replicationId; + } + @Override public ReplicationTimer getTimer() { - return timer; + return overallTimer; + } + + public List> getTimingData() { + return timingData; } public Stage getStage() { @@ -90,6 +118,12 @@ protected void validateAndSetStage(Stage expected, Stage next) { "can't move replication to stage [" + next + "]. current stage: [" + stage + "] (expected [" + expected + "])" ); } + // save the timing data for the current step + stageTimer.stop(); + timingData.add(new Tuple<>(stage.name(), stageTimer.time())); + // restart the step timer + stageTimer.reset(); + stageTimer.start(); stage = next; } @@ -97,16 +131,29 @@ public void setStage(Stage stage) { switch (stage) { case INIT: this.stage = Stage.INIT; - getIndex().reset(); break; case REPLICATING: validateAndSetStage(Stage.INIT, stage); - getIndex().start(); + // only start the overall timer once we've started replication + overallTimer.start(); break; - case DONE: + case GET_CHECKPOINT_INFO: validateAndSetStage(Stage.REPLICATING, stage); - getIndex().stop(); - getTimer().stop(); + break; + case FILE_DIFF: + validateAndSetStage(Stage.GET_CHECKPOINT_INFO, stage); + break; + case GET_FILES: + validateAndSetStage(Stage.FILE_DIFF, stage); + break; + case FINALIZE_REPLICATION: + validateAndSetStage(Stage.GET_FILES, stage); + break; + case DONE: + validateAndSetStage(Stage.FINALIZE_REPLICATION, stage); + // add the overall timing data + overallTimer.stop(); + timingData.add(new Tuple<>("OVERALL", overallTimer.time())); break; default: throw new IllegalArgumentException("unknown SegmentReplicationState.Stage [" + stage + "]"); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java index 73d9a2f805d75..a658ffc09d590 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTarget.java @@ -64,7 +64,7 @@ public SegmentReplicationTarget( super("replication_target", indexShard, new ReplicationLuceneIndex(), listener); this.checkpoint = checkpoint; this.source = source; - this.state = new SegmentReplicationState(stateIndex); + this.state = new SegmentReplicationState(stateIndex, getId()); this.multiFileWriter = new MultiFileWriter(indexShard.store(), stateIndex, getPrefix(), logger, this::ensureRefCount); } @@ -139,7 +139,9 @@ public void startReplication(ActionListener listener) { final StepListener getFilesListener = new StepListener<>(); final StepListener finalizeListener = new StepListener<>(); + logger.trace("[shardId {}] Replica starting replication [id {}]", shardId().getId(), getId()); // Get list of files to copy from this checkpoint. + state.setStage(SegmentReplicationState.Stage.GET_CHECKPOINT_INFO); source.getCheckpointMetadata(getId(), checkpoint, checkpointInfoListener); checkpointInfoListener.whenComplete(checkpointInfo -> getFiles(checkpointInfo, getFilesListener), listener::onFailure); @@ -152,14 +154,16 @@ public void startReplication(ActionListener listener) { private void getFiles(CheckpointInfoResponse checkpointInfo, StepListener getFilesListener) throws IOException { + state.setStage(SegmentReplicationState.Stage.FILE_DIFF); final Store.MetadataSnapshot snapshot = checkpointInfo.getSnapshot(); Store.MetadataSnapshot localMetadata = getMetadataSnapshot(); final Store.RecoveryDiff diff = snapshot.segmentReplicationDiff(localMetadata); - logger.debug("Replication diff {}", diff); - // Segments are immutable. So if the replica has any segments with the same name that differ from the one in the incoming snapshot - // from - // source that means the local copy of the segment has been corrupted/changed in some way and we throw an IllegalStateException to - // fail the shard + logger.trace("Replication diff {}", diff); + /* + * Segments are immutable. So if the replica has any segments with the same name that differ from the one in the incoming + * snapshot from source that means the local copy of the segment has been corrupted/changed in some way and we throw an + * IllegalStateException to fail the shard + */ if (diff.different.isEmpty() == false) { getFilesListener.onFailure( new IllegalStateException( @@ -177,15 +181,18 @@ private void getFiles(CheckpointInfoResponse checkpointInfo, StepListener listener) { + state.setStage(SegmentReplicationState.Stage.FINALIZE_REPLICATION); ActionListener.completeWith(listener, () -> { multiFileWriter.renameAllTempFiles(); final Store store = store(); diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java index f699f0edba842..a79ce195ad83b 100644 --- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java +++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java @@ -116,7 +116,7 @@ public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexSh * @param replicaShard replica shard on which checkpoint is received */ public synchronized void onNewCheckpoint(final ReplicationCheckpoint receivedCheckpoint, final IndexShard replicaShard) { - + logger.trace(() -> new ParameterizedMessage("Replica received new replication checkpoint from primary [{}]", receivedCheckpoint)); // Checks if received checkpoint is already present and ahead then it replaces old received checkpoint if (latestReceivedCheckpoint.get(replicaShard.shardId()) != null) { if (receivedCheckpoint.isAheadOf(latestReceivedCheckpoint.get(replicaShard.shardId()))) { @@ -139,6 +139,14 @@ public synchronized void onNewCheckpoint(final ReplicationCheckpoint receivedChe startReplication(receivedCheckpoint, replicaShard, new SegmentReplicationListener() { @Override public void onReplicationDone(SegmentReplicationState state) { + logger.trace( + () -> new ParameterizedMessage( + "[shardId {}] [replication id {}] Replication complete, timing data: {}", + replicaShard.shardId().getId(), + state.getReplicationId(), + state.getTimingData() + ) + ); // if we received a checkpoint during the copy event that is ahead of this // try and process it. if (latestReceivedCheckpoint.get(replicaShard.shardId()).isAheadOf(replicaShard.getLatestReplicationCheckpoint())) { @@ -154,6 +162,14 @@ public void onReplicationDone(SegmentReplicationState state) { @Override public void onReplicationFailure(SegmentReplicationState state, OpenSearchException e, boolean sendShardFailure) { + logger.trace( + () -> new ParameterizedMessage( + "[shardId {}] [replication id {}] Replication failed, timing data: {}", + replicaShard.shardId().getId(), + state.getReplicationId(), + state.getTimingData() + ) + ); if (sendShardFailure == true) { logger.error("replication failure", e); replicaShard.failShard("replication failure", e); @@ -172,9 +188,9 @@ public void startReplication( startReplication(new SegmentReplicationTarget(checkpoint, indexShard, sourceFactory.get(indexShard), listener)); } - public void startReplication(final SegmentReplicationTarget target) { + // pkg-private for integration tests + void startReplication(final SegmentReplicationTarget target) { final long replicationId = onGoingReplications.start(target, recoverySettings.activityTimeout()); - logger.trace(() -> new ParameterizedMessage("Starting replication {}", replicationId)); threadPool.generic().execute(new ReplicationRunner(replicationId)); } diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java index 8093b6aee88f9..cc51082639cdb 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java @@ -29,6 +29,7 @@ import org.opensearch.index.shard.IndexShardClosedException; import org.opensearch.indices.IndicesService; import org.opensearch.indices.replication.SegmentReplicationTargetService; +import org.opensearch.indices.replication.common.ReplicationTimer; import org.opensearch.node.NodeClosedException; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; @@ -103,7 +104,10 @@ final void publish(IndexShard indexShard) { // we have to execute under the system context so that if security is enabled the sync is authorized threadContext.markAsSystemContext(); PublishCheckpointRequest request = new PublishCheckpointRequest(indexShard.getLatestReplicationCheckpoint()); + final ReplicationCheckpoint checkpoint = request.getCheckpoint(); final ReplicationTask task = (ReplicationTask) taskManager.register("transport", "segrep_publish_checkpoint", request); + final ReplicationTimer timer = new ReplicationTimer(); + timer.start(); transportService.sendChildRequest( clusterService.localNode(), transportPrimaryAction, @@ -123,12 +127,23 @@ public String executor() { @Override public void handleResponse(ReplicationResponse response) { + timer.stop(); + logger.trace( + () -> new ParameterizedMessage( + "[shardId {}] Completed publishing checkpoint [{}], timing: {}", + indexShard.shardId().getId(), + checkpoint, + timer.time() + ) + ); task.setPhase("finished"); taskManager.unregister(task); } @Override public void handleException(TransportException e) { + timer.stop(); + logger.trace("[shardId {}] Failed to publish checkpoint, timing: {}", indexShard.shardId().getId(), timer.time()); task.setPhase("finished"); taskManager.unregister(task); if (ExceptionsHelper.unwrap(e, NodeClosedException.class) != null) { @@ -151,6 +166,13 @@ public void handleException(TransportException e) { } } ); + logger.trace( + () -> new ParameterizedMessage( + "[shardId {}] Publishing replication checkpoint [{}]", + checkpoint.getShardId().getId(), + checkpoint + ) + ); } } @@ -168,7 +190,7 @@ protected void shardOperationOnReplica(PublishCheckpointRequest request, IndexSh Objects.requireNonNull(request); Objects.requireNonNull(replica); ActionListener.completeWith(listener, () -> { - logger.trace("Checkpoint received on replica {}", request); + logger.trace(() -> new ParameterizedMessage("Checkpoint {} received on replica {}", request, replica.shardId())); if (request.getCheckpoint().getShardId().equals(replica.shardId())) { replicationService.onNewCheckpoint(request.getCheckpoint(), replica); } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java index 004fa7f614ef1..d3a6d1a97dacc 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java @@ -100,8 +100,8 @@ public void onReplicationFailure(SegmentReplicationState state, OpenSearchExcept ); final SegmentReplicationTarget spy = Mockito.spy(target); doAnswer(invocation -> { - // setting stage to REPLICATING so transition in markAsDone succeeds on listener completion - target.state().setStage(SegmentReplicationState.Stage.REPLICATING); + // set up stage correctly so the transition in markAsDone succeeds on listener completion + moveTargetToFinalStage(target); final ActionListener listener = invocation.getArgument(0); listener.onResponse(null); return null; @@ -123,7 +123,7 @@ public void onReplicationDone(SegmentReplicationState state) { @Override public void onReplicationFailure(SegmentReplicationState state, OpenSearchException e, boolean sendShardFailure) { - assertEquals(SegmentReplicationState.Stage.REPLICATING, state.getStage()); + assertEquals(SegmentReplicationState.Stage.INIT, state.getStage()); assertEquals(expectedError, e.getCause()); assertTrue(sendShardFailure); } @@ -131,8 +131,6 @@ public void onReplicationFailure(SegmentReplicationState state, OpenSearchExcept ); final SegmentReplicationTarget spy = Mockito.spy(target); doAnswer(invocation -> { - // setting stage to REPLICATING so transition in markAsDone succeeds on listener completion - target.state().setStage(SegmentReplicationState.Stage.REPLICATING); final ActionListener listener = invocation.getArgument(0); listener.onFailure(expectedError); return null; @@ -271,4 +269,17 @@ public void testBeforeIndexShardClosed_CancelsOngoingReplications() { sut.beforeIndexShardClosed(indexShard.shardId(), indexShard, Settings.EMPTY); verify(spy, times(1)).cancel(any()); } + + /** + * Move the {@link SegmentReplicationTarget} object through its {@link SegmentReplicationState.Stage} values in order + * until the final, non-terminal stage. + */ + private void moveTargetToFinalStage(SegmentReplicationTarget target) { + SegmentReplicationState.Stage[] stageValues = SegmentReplicationState.Stage.values(); + assertEquals(target.state().getStage(), SegmentReplicationState.Stage.INIT); + // Skip the first two stages (DONE and INIT) and iterate until the last value + for (int i = 2; i < stageValues.length; i++) { + target.state().setStage(stageValues[i]); + } + } } diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java index 1157c463785ac..11217a46b3c69 100644 --- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java +++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetTests.java @@ -28,6 +28,7 @@ import org.junit.Assert; import org.mockito.Mockito; import org.opensearch.ExceptionsHelper; +import org.opensearch.OpenSearchException; import org.opensearch.action.ActionListener; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; @@ -96,7 +97,7 @@ public class SegmentReplicationTargetTests extends IndexShardTestCase { Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, org.opensearch.Version.CURRENT).build() ); - SegmentInfos testSegmentInfos; + private SegmentInfos testSegmentInfos; @Override public void setUp() throws Exception { @@ -162,6 +163,7 @@ public void getSegmentFiles( public void onResponse(Void replicationResponse) { try { verify(spyIndexShard, times(1)).finalizeReplication(any(), anyLong()); + segrepTarget.markAsDone(); } catch (IOException ex) { Assert.fail(); } @@ -169,7 +171,7 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { - logger.error("Unexpected test error", e); + logger.error("Unexpected onFailure", e); Assert.fail(); } }); @@ -213,6 +215,7 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { assertEquals(exception, e.getCause().getCause()); + segrepTarget.fail(new OpenSearchException(e), false); } }); } @@ -255,6 +258,7 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { assertEquals(exception, e.getCause().getCause()); + segrepTarget.fail(new OpenSearchException(e), false); } }); } @@ -299,6 +303,7 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { assertEquals(exception, e.getCause()); + segrepTarget.fail(new OpenSearchException(e), false); } }); } @@ -343,6 +348,7 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { assertEquals(exception, e.getCause()); + segrepTarget.fail(new OpenSearchException(e), false); } }); } @@ -384,6 +390,7 @@ public void onResponse(Void replicationResponse) { @Override public void onFailure(Exception e) { assert (e instanceof IllegalStateException); + segrepTarget.fail(new OpenSearchException(e), false); } }); } @@ -432,11 +439,13 @@ public void getSegmentFiles( @Override public void onResponse(Void replicationResponse) { logger.info("No error processing checkpoint info"); + segrepTarget.markAsDone(); } @Override public void onFailure(Exception e) { - assert (e instanceof IllegalStateException); + logger.error("Unexpected onFailure", e); + Assert.fail(); } }); } @@ -448,7 +457,7 @@ public void onFailure(Exception e) { * @return * @throws IOException */ - List generateStoreMetadataSnapshot(int docCount) throws IOException { + private List generateStoreMetadataSnapshot(int docCount) throws IOException { List docList = new ArrayList<>(); for (int i = 0; i < docCount; i++) { Document document = new Document(); @@ -480,7 +489,7 @@ List generateStoreMetadataSnapshot(int docCount) throws return Arrays.asList(storeMetadata, storeMetadataWithDeletes); } - public static void deleteContent(Directory directory) throws IOException { + private static void deleteContent(Directory directory) throws IOException { final String[] files = directory.listAll(); final List exceptions = new ArrayList<>(); for (String file : files) { @@ -498,7 +507,6 @@ public static void deleteContent(Directory directory) throws IOException { @Override public void tearDown() throws Exception { super.tearDown(); - segrepTarget.markAsDone(); closeShards(spyIndexShard, indexShard); } } From 36f1d77ad7983d9a58bfd755423ce806941fd930 Mon Sep 17 00:00:00 2001 From: Owais Kazi Date: Wed, 17 Aug 2022 17:14:09 -0700 Subject: [PATCH 22/27] Handles the status code for `.` properties (#4246) * Return 400 status code for array out of bound Signed-off-by: Owais Kazi * Spotless apply Signed-off-by: Owais Kazi * PR comments Signed-off-by: Owais Kazi Signed-off-by: Owais Kazi --- .../opensearch/index/mapper/ObjectMapper.java | 4 ++++ .../index/mapper/ObjectMapperTests.java | 20 +++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index fcbca6049ec0b..1702c7700cf60 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -380,6 +380,10 @@ protected static void parseProperties(ObjectMapper.Builder objBuilder, Map fieldBuilder = typeParser.parse(realFieldName, propNode, parserContext); for (int i = fieldNameParts.length - 2; i >= 0; --i) { diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java index 079475d9f3554..d6c89342c9df2 100644 --- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java @@ -178,6 +178,26 @@ public void testFieldsWithFilledArrayShouldThrowException() throws Exception { } } + public void testDotAsFieldName() throws Exception { + String mapping = Strings.toString( + XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(".") + .field("type", "text") + .endObject() + .endObject() + .endObject() + ); + + try { + createIndex("test").mapperService().documentMapperParser().parse("tweet", new CompressedXContent(mapping)); + fail("Expected MapperParsingException"); + } catch (MapperParsingException e) { + assertThat(e.getMessage(), containsString("Invalid field name")); + } + } + public void testFieldPropertiesArray() throws Exception { String mapping = Strings.toString( XContentFactory.jsonBuilder() From d308a2925818ad89e0a85161bc6ee43c057eb6d8 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Wed, 17 Aug 2022 17:25:19 -0700 Subject: [PATCH 23/27] [Segment Replication] Update PrimaryShardAllocator to prefer replicas with higher replication checkpoint (#4041) * [Segment Replication] Update PrimaryShardAllocator to prefer replicas having higher replication checkpoint Signed-off-by: Suraj Singh * Use empty replication checkpoint to avoid NPE Signed-off-by: Suraj Singh * Update NodeGatewayStartedShards to optionally wire in/out ReplicationCheckpoint field Signed-off-by: Suraj Singh * Use default replication checkpoint causing EOF errors on empty checkpoint * Add indexSettings to GatewayAllocator to allow ReplicationCheckpoint comparator only for segrep enabled indices * Add unit tests for primary term first replica promotion & comparator fix * Fix NPE on empty IndexMetadata * Remove settings from AllocationService and directly inject in GatewayAllocator * Add more unit tests and minor code clean up Signed-off-by: Suraj Singh * Address review comments & integration test Signed-off-by: Suraj Singh * Fix comparator on null ReplicationCheckpoint Signed-off-by: Suraj Singh Signed-off-by: Suraj Singh --- .../gateway/PrimaryShardAllocator.java | 17 +- ...ransportNodesListGatewayStartedShards.java | 58 ++++- .../checkpoint/ReplicationCheckpoint.java | 7 +- .../gateway/PrimaryShardAllocatorTests.java | 229 +++++++++++++++++- .../test/gateway/TestGatewayAllocator.java | 17 +- 5 files changed, 314 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/opensearch/gateway/PrimaryShardAllocator.java b/server/src/main/java/org/opensearch/gateway/PrimaryShardAllocator.java index b9cfebaa98521..4dc9396751fc9 100644 --- a/server/src/main/java/org/opensearch/gateway/PrimaryShardAllocator.java +++ b/server/src/main/java/org/opensearch/gateway/PrimaryShardAllocator.java @@ -313,6 +313,11 @@ private static ShardStoreInfo shardStoreInfo(NodeGatewayStartedShards nodeShardS NodeGatewayStartedShards::primary ).reversed(); + private static final Comparator HIGHEST_REPLICATION_CHECKPOINT_FIRST_COMPARATOR = Comparator.comparing( + NodeGatewayStartedShards::replicationCheckpoint, + Comparator.nullsLast(Comparator.naturalOrder()) + ); + /** * Builds a list of nodes. If matchAnyShard is set to false, only nodes that have an allocation id matching * inSyncAllocationIds are added to the list. Otherwise, any node that has a shard is added to the list, but @@ -381,6 +386,12 @@ protected static NodeShardsResult buildNodeShardsResult( } } + /** + * Orders the active shards copies based on below comparators + * 1. No store exception i.e. shard copy is readable + * 2. Prefer previous primary shard + * 3. Prefer shard copy with the highest replication checkpoint. It is NO-OP for doc rep enabled indices. + */ final Comparator comparator; // allocation preference if (matchAnyShard) { // prefer shards with matching allocation ids @@ -388,9 +399,11 @@ protected static NodeShardsResult buildNodeShardsResult( (NodeGatewayStartedShards state) -> inSyncAllocationIds.contains(state.allocationId()) ).reversed(); comparator = matchingAllocationsFirst.thenComparing(NO_STORE_EXCEPTION_FIRST_COMPARATOR) - .thenComparing(PRIMARY_FIRST_COMPARATOR); + .thenComparing(PRIMARY_FIRST_COMPARATOR) + .thenComparing(HIGHEST_REPLICATION_CHECKPOINT_FIRST_COMPARATOR); } else { - comparator = NO_STORE_EXCEPTION_FIRST_COMPARATOR.thenComparing(PRIMARY_FIRST_COMPARATOR); + comparator = NO_STORE_EXCEPTION_FIRST_COMPARATOR.thenComparing(PRIMARY_FIRST_COMPARATOR) + .thenComparing(HIGHEST_REPLICATION_CHECKPOINT_FIRST_COMPARATOR); } nodeShardStates.sort(comparator); diff --git a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java index 78b4fa287ef59..953b4def9d653 100644 --- a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java +++ b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java @@ -35,6 +35,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.LegacyESVersion; import org.opensearch.OpenSearchException; +import org.opensearch.Version; import org.opensearch.action.ActionListener; import org.opensearch.action.ActionType; import org.opensearch.action.FailedNodeException; @@ -56,11 +57,13 @@ import org.opensearch.common.xcontent.NamedXContentRegistry; import org.opensearch.env.NodeEnvironment; import org.opensearch.index.IndexSettings; +import org.opensearch.index.shard.IndexShard; import org.opensearch.index.shard.ShardId; import org.opensearch.index.shard.ShardPath; import org.opensearch.index.shard.ShardStateMetadata; import org.opensearch.index.store.Store; import org.opensearch.indices.IndicesService; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; @@ -195,6 +198,7 @@ protected NodeGatewayStartedShards nodeOperation(NodeRequest request) { clusterService.localNode(), allocationId, shardStateMetadata.primary, + null, exception ); } @@ -202,10 +206,16 @@ protected NodeGatewayStartedShards nodeOperation(NodeRequest request) { logger.debug("{} shard state info found: [{}]", shardId, shardStateMetadata); String allocationId = shardStateMetadata.allocationId != null ? shardStateMetadata.allocationId.getId() : null; - return new NodeGatewayStartedShards(clusterService.localNode(), allocationId, shardStateMetadata.primary); + final IndexShard shard = indicesService.getShardOrNull(shardId); + return new NodeGatewayStartedShards( + clusterService.localNode(), + allocationId, + shardStateMetadata.primary, + shard != null ? shard.getLatestReplicationCheckpoint() : null + ); } logger.trace("{} no local shard info found", shardId); - return new NodeGatewayStartedShards(clusterService.localNode(), null, false); + return new NodeGatewayStartedShards(clusterService.localNode(), null, false, null); } catch (Exception e) { throw new OpenSearchException("failed to load started shards", e); } @@ -349,10 +359,10 @@ public String getCustomDataPath() { * @opensearch.internal */ public static class NodeGatewayStartedShards extends BaseNodeResponse { - private final String allocationId; private final boolean primary; private final Exception storeException; + private final ReplicationCheckpoint replicationCheckpoint; public NodeGatewayStartedShards(StreamInput in) throws IOException { super(in); @@ -363,16 +373,33 @@ public NodeGatewayStartedShards(StreamInput in) throws IOException { } else { storeException = null; } + if (in.getVersion().onOrAfter(Version.V_3_0_0) && in.readBoolean()) { + replicationCheckpoint = new ReplicationCheckpoint(in); + } else { + replicationCheckpoint = null; + } } - public NodeGatewayStartedShards(DiscoveryNode node, String allocationId, boolean primary) { - this(node, allocationId, primary, null); + public NodeGatewayStartedShards( + DiscoveryNode node, + String allocationId, + boolean primary, + ReplicationCheckpoint replicationCheckpoint + ) { + this(node, allocationId, primary, replicationCheckpoint, null); } - public NodeGatewayStartedShards(DiscoveryNode node, String allocationId, boolean primary, Exception storeException) { + public NodeGatewayStartedShards( + DiscoveryNode node, + String allocationId, + boolean primary, + ReplicationCheckpoint replicationCheckpoint, + Exception storeException + ) { super(node); this.allocationId = allocationId; this.primary = primary; + this.replicationCheckpoint = replicationCheckpoint; this.storeException = storeException; } @@ -384,6 +411,10 @@ public boolean primary() { return this.primary; } + public ReplicationCheckpoint replicationCheckpoint() { + return this.replicationCheckpoint; + } + public Exception storeException() { return this.storeException; } @@ -399,6 +430,14 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeBoolean(false); } + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (replicationCheckpoint != null) { + out.writeBoolean(true); + replicationCheckpoint.writeTo(out); + } else { + out.writeBoolean(false); + } + } } @Override @@ -414,7 +453,8 @@ public boolean equals(Object o) { return primary == that.primary && Objects.equals(allocationId, that.allocationId) - && Objects.equals(storeException, that.storeException); + && Objects.equals(storeException, that.storeException) + && Objects.equals(replicationCheckpoint, that.replicationCheckpoint); } @Override @@ -422,6 +462,7 @@ public int hashCode() { int result = (allocationId != null ? allocationId.hashCode() : 0); result = 31 * result + (primary ? 1 : 0); result = 31 * result + (storeException != null ? storeException.hashCode() : 0); + result = 31 * result + (replicationCheckpoint != null ? replicationCheckpoint.hashCode() : 0); return result; } @@ -432,6 +473,9 @@ public String toString() { if (storeException != null) { buf.append(",storeException=").append(storeException); } + if (replicationCheckpoint != null) { + buf.append(",ReplicationCheckpoint=").append(replicationCheckpoint.toString()); + } buf.append("]"); return buf.toString(); } diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java index 8afb5bd055636..6a4e5e449f178 100644 --- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java +++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/ReplicationCheckpoint.java @@ -23,7 +23,7 @@ * * @opensearch.internal */ -public class ReplicationCheckpoint implements Writeable { +public class ReplicationCheckpoint implements Writeable, Comparable { private final ShardId shardId; private final long primaryTerm; @@ -107,6 +107,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(segmentInfosVersion); } + @Override + public int compareTo(ReplicationCheckpoint other) { + return this.isAheadOf(other) ? -1 : 1; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java b/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java index 4a1ecb9661687..3c39ec9f03b2a 100644 --- a/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java +++ b/server/src/test/java/org/opensearch/gateway/PrimaryShardAllocatorTests.java @@ -62,6 +62,7 @@ import org.opensearch.common.util.set.Sets; import org.opensearch.env.ShardLockObtainFailedException; import org.opensearch.index.shard.ShardId; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.repositories.IndexId; import org.opensearch.snapshots.Snapshot; import org.opensearch.snapshots.SnapshotId; @@ -205,6 +206,203 @@ public void testShardLockObtainFailedException() { assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); } + /** + * Tests that replica with the highest primary term version will be selected as target + */ + public void testPreferReplicaWithHighestPrimaryTerm() { + String allocId1 = randomAlphaOfLength(10); + String allocId2 = randomAlphaOfLength(10); + String allocId3 = randomAlphaOfLength(10); + final RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas( + yesAllocationDeciders(), + CLUSTER_RECOVERED, + allocId1, + allocId2, + allocId3 + ); + testAllocator.addData(node1, allocId1, false, new ReplicationCheckpoint(shardId, 20, 10, 101, 1)); + testAllocator.addData(node2, allocId2, false, new ReplicationCheckpoint(shardId, 22, 10, 120, 2)); + testAllocator.addData(node3, allocId3, false, new ReplicationCheckpoint(shardId, 20, 10, 120, 2)); + allocateAllUnassigned(allocation); + assertThat(allocation.routingNodesChanged(), equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), + equalTo(node2.getId()) + ); + // Assert node2's allocation id is used + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), + equalTo(allocId2) + ); + assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); + } + + /** + * Tests that replica with highest primary ter version will be selected as target + */ + public void testPreferReplicaWithNullReplicationCheckpoint() { + String allocId1 = randomAlphaOfLength(10); + String allocId2 = randomAlphaOfLength(10); + String allocId3 = randomAlphaOfLength(10); + final RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas( + yesAllocationDeciders(), + CLUSTER_RECOVERED, + allocId1, + allocId2, + allocId3 + ); + testAllocator.addData(node1, allocId1, false, new ReplicationCheckpoint(shardId, 20, 10, 101, 1)); + testAllocator.addData(node2, allocId2, false); + testAllocator.addData(node3, allocId3, false, new ReplicationCheckpoint(shardId, 40, 10, 120, 2)); + allocateAllUnassigned(allocation); + assertThat(allocation.routingNodesChanged(), equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), + equalTo(node3.getId()) + ); + // Assert node3's allocation id should be used as it has highest replication checkpoint + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), + equalTo(allocId3) + ); + assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); + } + + /** + * Tests that null ReplicationCheckpoint are ignored + */ + public void testPreferReplicaWithAllNullReplicationCheckpoint() { + String allocId1 = randomAlphaOfLength(10); + String allocId2 = randomAlphaOfLength(10); + String allocId3 = randomAlphaOfLength(10); + final RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas( + yesAllocationDeciders(), + CLUSTER_RECOVERED, + allocId1, + allocId2, + allocId3 + ); + testAllocator.addData(node1, allocId1, false, null, null); + testAllocator.addData(node2, allocId2, false, null, null); + testAllocator.addData(node3, allocId3, true, null, null); + allocateAllUnassigned(allocation); + assertThat(allocation.routingNodesChanged(), equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), + equalTo(node3.getId()) + ); + // Assert node3's allocation id should be used as it was previous primary + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), + equalTo(allocId3) + ); + assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); + } + + /** + * Tests that replica with highest segment info version will be selected as target on equal primary terms + */ + public void testPreferReplicaWithHighestSegmentInfoVersion() { + String allocId1 = randomAlphaOfLength(10); + String allocId2 = randomAlphaOfLength(10); + String allocId3 = randomAlphaOfLength(10); + final RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas( + yesAllocationDeciders(), + CLUSTER_RECOVERED, + allocId1, + allocId2, + allocId3 + ); + testAllocator.addData(node1, allocId1, false, new ReplicationCheckpoint(shardId, 10, 10, 101, 1)); + testAllocator.addData(node2, allocId2, false, new ReplicationCheckpoint(shardId, 20, 10, 120, 3)); + testAllocator.addData(node3, allocId3, false, new ReplicationCheckpoint(shardId, 20, 10, 120, 2)); + allocateAllUnassigned(allocation); + assertThat(allocation.routingNodesChanged(), equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), + equalTo(node2.getId()) + ); + // Assert node2's allocation id is used + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), + equalTo(allocId2) + ); + assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); + } + + /** + * Tests that prefer allocation of replica at lower checkpoint but in sync set + */ + public void testOutOfSyncHighestRepCheckpointIsIgnored() { + String allocId1 = randomAlphaOfLength(10); + String allocId2 = randomAlphaOfLength(10); + String allocId3 = randomAlphaOfLength(10); + final RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas( + yesAllocationDeciders(), + CLUSTER_RECOVERED, + allocId1, + allocId3 + ); + testAllocator.addData(node1, allocId1, false, new ReplicationCheckpoint(shardId, 10, 10, 101, 1)); + testAllocator.addData(node2, allocId2, false, new ReplicationCheckpoint(shardId, 20, 10, 120, 2)); + testAllocator.addData(node3, allocId3, false, new ReplicationCheckpoint(shardId, 15, 10, 120, 2)); + allocateAllUnassigned(allocation); + assertThat(allocation.routingNodesChanged(), equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), + equalTo(node3.getId()) + ); + // Assert node3's allocation id is used + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), + equalTo(allocId3) + ); + assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); + } + + /** + * Tests that prefer allocation of older primary over replica with higher replication checkpoint + */ + public void testPreferAllocatingPreviousPrimaryWithLowerRepCheckpoint() { + String allocId1 = randomAlphaOfLength(10); + String allocId2 = randomAlphaOfLength(10); + String allocId3 = randomAlphaOfLength(10); + final RoutingAllocation allocation = routingAllocationWithOnePrimaryNoReplicas( + yesAllocationDeciders(), + CLUSTER_RECOVERED, + allocId1, + allocId2, + allocId3 + ); + testAllocator.addData(node1, allocId1, true, new ReplicationCheckpoint(shardId, 10, 10, 101, 1)); + testAllocator.addData(node2, allocId2, false, new ReplicationCheckpoint(shardId, 20, 10, 120, 2)); + testAllocator.addData(node3, allocId3, false, new ReplicationCheckpoint(shardId, 15, 10, 120, 2)); + allocateAllUnassigned(allocation); + assertThat(allocation.routingNodesChanged(), equalTo(true)); + assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); + assertThat(allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(1)); + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), + equalTo(node1.getId()) + ); + // Assert node1's allocation id is used with highest replication checkpoint + assertThat( + allocation.routingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).allocationId().getId(), + equalTo(allocId1) + ); + assertClusterHealthStatus(allocation, ClusterHealthStatus.YELLOW); + } + /** * Tests that when one node returns a ShardLockObtainFailedException and another properly loads the store, it will * select the second node as target @@ -219,7 +417,7 @@ public void testShardLockObtainFailedExceptionPreferOtherValidCopies() { allocId2 ); testAllocator.addData(node1, allocId1, randomBoolean(), new ShardLockObtainFailedException(shardId, "test")); - testAllocator.addData(node2, allocId2, randomBoolean(), null); + testAllocator.addData(node2, allocId2, randomBoolean()); allocateAllUnassigned(allocation); assertThat(allocation.routingNodesChanged(), equalTo(true)); assertThat(allocation.routingNodes().unassigned().ignored().isEmpty(), equalTo(true)); @@ -601,17 +799,42 @@ public TestAllocator clear() { return this; } + public TestAllocator addData( + DiscoveryNode node, + String allocationId, + boolean primary, + ReplicationCheckpoint replicationCheckpoint + ) { + return addData(node, allocationId, primary, replicationCheckpoint, null); + } + public TestAllocator addData(DiscoveryNode node, String allocationId, boolean primary) { - return addData(node, allocationId, primary, null); + return addData(node, allocationId, primary, ReplicationCheckpoint.empty(shardId), null); } public TestAllocator addData(DiscoveryNode node, String allocationId, boolean primary, @Nullable Exception storeException) { + return addData(node, allocationId, primary, ReplicationCheckpoint.empty(shardId), storeException); + } + + public TestAllocator addData( + DiscoveryNode node, + String allocationId, + boolean primary, + ReplicationCheckpoint replicationCheckpoint, + @Nullable Exception storeException + ) { if (data == null) { data = new HashMap<>(); } data.put( node, - new TransportNodesListGatewayStartedShards.NodeGatewayStartedShards(node, allocationId, primary, storeException) + new TransportNodesListGatewayStartedShards.NodeGatewayStartedShards( + node, + allocationId, + primary, + replicationCheckpoint, + storeException + ) ); return this; } diff --git a/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java b/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java index 54c92f4d519aa..a36dc26685eb4 100644 --- a/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java +++ b/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java @@ -43,6 +43,7 @@ import org.opensearch.gateway.ReplicaShardAllocator; import org.opensearch.gateway.TransportNodesListGatewayStartedShards.NodeGatewayStartedShards; import org.opensearch.index.shard.ShardId; +import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint; import org.opensearch.indices.store.TransportNodesListShardStoreMetadata.NodeStoreFilesMetadata; import java.util.Collections; @@ -71,6 +72,7 @@ public class TestGatewayAllocator extends GatewayAllocator { Map> knownAllocations = new HashMap<>(); DiscoveryNodes currentNodes = DiscoveryNodes.EMPTY_NODES; + Map shardIdNodeToReplicationCheckPointMap = new HashMap<>(); PrimaryShardAllocator primaryShardAllocator = new PrimaryShardAllocator() { @Override @@ -90,7 +92,8 @@ protected AsyncShardFetch.FetchResult fetchData(ShardR routing -> new NodeGatewayStartedShards( currentNodes.get(routing.currentNodeId()), routing.allocationId().getId(), - routing.primary() + routing.primary(), + getReplicationCheckpoint(shardId, routing.currentNodeId()) ) ) ); @@ -99,6 +102,10 @@ protected AsyncShardFetch.FetchResult fetchData(ShardR } }; + private ReplicationCheckpoint getReplicationCheckpoint(ShardId shardId, String nodeName) { + return shardIdNodeToReplicationCheckPointMap.getOrDefault(getReplicationCheckPointKey(shardId, nodeName), null); + } + ReplicaShardAllocator replicaShardAllocator = new ReplicaShardAllocator() { @Override protected AsyncShardFetch.FetchResult fetchData(ShardRouting shard, RoutingAllocation allocation) { @@ -156,4 +163,12 @@ public void allocateUnassigned( public void addKnownAllocation(ShardRouting shard) { knownAllocations.computeIfAbsent(shard.currentNodeId(), id -> new HashMap<>()).put(shard.shardId(), shard); } + + public String getReplicationCheckPointKey(ShardId shardId, String nodeName) { + return shardId.toString() + "_" + nodeName; + } + + public void addReplicationCheckpoint(ShardId shardId, String nodeName, ReplicationCheckpoint replicationCheckpoint) { + shardIdNodeToReplicationCheckPointMap.putIfAbsent(getReplicationCheckPointKey(shardId, nodeName), replicationCheckpoint); + } } From ee26e010fcb074f073eaa6bdeeb563c42b382825 Mon Sep 17 00:00:00 2001 From: Andriy Redko Date: Thu, 18 Aug 2022 15:51:26 -0400 Subject: [PATCH 24/27] Fixing NodeGatewayStartedShards bwc (de)serialization issues (#4258) Signed-off-by: Andriy Redko Signed-off-by: Andriy Redko --- modules/repository-s3/licenses/commons-logging-1.2.jar.sha1 | 1 - .../gateway/TransportNodesListGatewayStartedShards.java | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 modules/repository-s3/licenses/commons-logging-1.2.jar.sha1 diff --git a/modules/repository-s3/licenses/commons-logging-1.2.jar.sha1 b/modules/repository-s3/licenses/commons-logging-1.2.jar.sha1 deleted file mode 100644 index f40f0242448e8..0000000000000 --- a/modules/repository-s3/licenses/commons-logging-1.2.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -4bfc12adfe4842bf07b657f0369c4cb522955686 \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java index 953b4def9d653..c43f539243d7a 100644 --- a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java +++ b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShards.java @@ -373,7 +373,7 @@ public NodeGatewayStartedShards(StreamInput in) throws IOException { } else { storeException = null; } - if (in.getVersion().onOrAfter(Version.V_3_0_0) && in.readBoolean()) { + if (in.getVersion().onOrAfter(Version.V_2_3_0) && in.readBoolean()) { replicationCheckpoint = new ReplicationCheckpoint(in); } else { replicationCheckpoint = null; @@ -430,7 +430,7 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeBoolean(false); } - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_3_0)) { if (replicationCheckpoint != null) { out.writeBoolean(true); replicationCheckpoint.writeTo(out); From 4643620eda3818ef63c474f90dde6814128101c8 Mon Sep 17 00:00:00 2001 From: Daniel Widdis Date: Sun, 21 Aug 2022 10:52:20 -0700 Subject: [PATCH 25/27] Fix AbstractStringFieldDataTestCase tests to account for TotalHits lower bound (#4270) Fixes tests to account for TotalHits uncertainty as of Lucene 9. Signed-off-by: Daniel Widdis --- .../index/fielddata/AbstractStringFieldDataTestCase.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/index/fielddata/AbstractStringFieldDataTestCase.java b/server/src/test/java/org/opensearch/index/fielddata/AbstractStringFieldDataTestCase.java index 763ee59a385a2..76496491b3ed4 100644 --- a/server/src/test/java/org/opensearch/index/fielddata/AbstractStringFieldDataTestCase.java +++ b/server/src/test/java/org/opensearch/index/fielddata/AbstractStringFieldDataTestCase.java @@ -52,6 +52,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.join.QueryBitSetProducer; import org.apache.lucene.search.join.ScoreMode; import org.apache.lucene.search.join.ToParentBlockJoinQuery; @@ -340,7 +341,13 @@ public void testSortMissing(boolean first, boolean reverse) throws IOException { randomBoolean() ? numDocs : randomIntBetween(10, numDocs), new Sort(sortField) ); - assertEquals(numDocs, topDocs.totalHits.value); + // As of Lucene 9.0.0, totalHits may be a lower bound + if (topDocs.totalHits.relation == TotalHits.Relation.EQUAL_TO) { + assertEquals(numDocs, topDocs.totalHits.value); + } else { + assertTrue(1000 <= topDocs.totalHits.value); + assertTrue(numDocs >= topDocs.totalHits.value); + } BytesRef previousValue = first ? null : reverse ? UnicodeUtil.BIG_TERM : new BytesRef(); for (int i = 0; i < topDocs.scoreDocs.length; ++i) { final String docValue = searcher.doc(topDocs.scoreDocs[i].doc).get("value"); From 6629b09aad2629fafb7aad998153867c2f3fe472 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 09:40:48 -0400 Subject: [PATCH 26/27] Bump com.gradle.enterprise from 3.10.3 to 3.11.1 (#4273) Bumps com.gradle.enterprise from 3.10.3 to 3.11.1. --- updated-dependencies: - dependency-name: com.gradle.enterprise dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- settings.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.gradle b/settings.gradle index 65dc6a13100e2..4c389b5490e7c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -10,7 +10,7 @@ */ plugins { - id "com.gradle.enterprise" version "3.10.3" + id "com.gradle.enterprise" version "3.11.1" } buildCache { From 54330560a7064434e088dad0eeb61ad0df2cfa6b Mon Sep 17 00:00:00 2001 From: Navneet Verma Date: Mon, 22 Aug 2022 13:03:22 -0700 Subject: [PATCH 27/27] Refactored the src and test of GeoHashGrid and GeoTileGrid Aggregations on GeoPoint from server folder to geo module.(#4071) (#4072) (#4180) The changes also includes: * Updated Search plugin to provide the interface so that plugins can also register the composite aggregations * Added YAML test for the geo_grid, geo_tile and composite aggregation Signed-off-by: Navneet Verma --- .../client/RestHighLevelClient.java | 6 - modules/geo/build.gradle | 9 +- .../aggregations/bucket/GeoHashGridIT.java | 39 +- .../aggregations/bucket/ShardReduceIT.java | 107 +++++ ...actGeoAggregatorModulePluginTestCase.java} | 2 +- ...BoundsIT.java => GeoBoundsITTestCase.java} | 2 +- .../metrics/GeoCentroidITTestCase.java | 84 ++++ .../org/opensearch/geo/GeoModulePlugin.java | 42 +- .../GeoTileGridValuesSourceBuilder.java | 29 +- .../bucket/composite/GeoTileValuesSource.java | 8 +- .../bucket/geogrid/BoundedCellValues.java | 2 +- .../bucket/geogrid/BucketPriorityQueue.java | 2 +- .../bucket/geogrid/CellIdSource.java | 2 +- .../bucket/geogrid/CellValues.java | 2 +- .../aggregations/bucket/geogrid/GeoGrid.java | 2 +- .../geogrid/GeoGridAggregationBuilder.java | 2 +- .../bucket/geogrid/GeoGridAggregator.java | 2 +- .../GeoHashGridAggregationBuilder.java | 4 +- .../bucket/geogrid/GeoHashGridAggregator.java | 2 +- .../geogrid/GeoHashGridAggregatorFactory.java | 2 +- .../GeoTileGridAggregationBuilder.java | 5 +- .../bucket/geogrid/GeoTileGridAggregator.java | 2 +- .../geogrid/GeoTileGridAggregatorFactory.java | 3 +- .../bucket/geogrid/InternalGeoGrid.java | 2 +- .../bucket/geogrid/InternalGeoGridBucket.java | 2 +- .../bucket/geogrid/InternalGeoHashGrid.java | 2 +- .../geogrid/InternalGeoHashGridBucket.java | 2 +- .../bucket/geogrid/InternalGeoTileGrid.java | 2 +- .../geogrid/InternalGeoTileGridBucket.java | 3 +- .../bucket/geogrid/ParsedGeoGrid.java | 2 +- .../bucket/geogrid/ParsedGeoGridBucket.java | 2 +- .../bucket/geogrid/ParsedGeoHashGrid.java | 2 +- .../geogrid/ParsedGeoHashGridBucket.java | 2 +- .../bucket/geogrid/ParsedGeoTileGrid.java | 2 +- .../geogrid/ParsedGeoTileGridBucket.java | 3 +- .../bucket/geogrid/UnboundedCellValues.java | 2 +- .../bucket/geogrid/package-info.java | 2 +- .../metrics/GeoGridAggregatorSupplier.java | 4 +- .../GeoHashGridAggregationBuilderTests.java | 21 +- .../GeoTileGridAggregationBuilderTests.java | 23 +- ...idAggregationCompositeAggregatorTests.java | 174 ++++++++ ...eGridCompositeAggregationBuilderTests.java | 50 +++ .../GeoTileGridValuesSourceBuilderTests.java | 3 +- .../geogrid/GeoGridAggregatorTestCase.java | 20 +- .../bucket/geogrid/GeoGridTestCase.java | 39 +- .../geogrid/GeoHashGridAggregatorTests.java | 2 +- .../geogrid/GeoHashGridParserTests.java | 2 +- .../bucket/geogrid/GeoHashGridTests.java | 2 +- .../geogrid/GeoTileGridAggregatorTests.java | 4 +- .../geogrid/GeoTileGridParserTests.java | 3 +- .../bucket/geogrid/GeoTileGridTests.java | 3 +- .../geo/tests/common/AggregationBuilders.java | 18 + .../common/AggregationInspectionHelper.java | 5 + .../geo/tests/common/RandomGeoGenerator.java | 11 + .../test/geo_shape/230_composite.yml | 168 ++++++++ .../test/geo_shape}/280_geohash_grid.yml | 0 .../test/geo_shape}/290_geotile_grid.yml | 0 .../test/search.aggregation/230_composite.yml | 86 ---- .../aggregations/bucket/ShardReduceIT.java | 35 -- .../aggregations/metrics/GeoCentroidIT.java | 33 -- .../org/opensearch/plugins/SearchPlugin.java | 82 ++++ .../org/opensearch/search/DocValueFormat.java | 2 +- .../org/opensearch/search/SearchModule.java | 22 +- .../aggregations/AggregationBuilders.java | 18 - .../bucket/{geogrid => }/GeoTileUtils.java | 8 +- .../CompositeAggregationBuilder.java | 50 ++- .../CompositeAggregationParsingFunction.java | 22 + .../CompositeValuesSourceBuilder.java | 6 +- .../CompositeValuesSourceConfig.java | 18 +- .../CompositeValuesSourceParserHelper.java | 78 +++- .../bucket/composite/LongValuesSource.java | 6 +- .../SingleDimensionValuesSource.java | 2 +- .../support/AggregationInspectionHelper.java | 5 - .../aggregations/support/package-info.java | 2 +- .../search/DocValueFormatTests.java | 2 +- .../aggregations/AggregationsTests.java | 4 - .../CompositeAggregationBuilderTests.java | 20 +- .../composite/CompositeAggregatorTests.java | 384 +----------------- .../bucket/geogrid/GeoTileUtilsTests.java | 13 +- .../BaseCompositeAggregatorTestCase.java | 310 ++++++++++++++ .../test/InternalAggregationTestCase.java | 6 - 81 files changed, 1395 insertions(+), 761 deletions(-) rename {server/src/internalClusterTest/java/org/opensearch => modules/geo/src/internalClusterTest/java/org/opensearch/geo}/search/aggregations/bucket/GeoHashGridIT.java (89%) create mode 100644 modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java rename modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/{AbstractGeoAggregatorTestCaseModulePlugin.java => AbstractGeoAggregatorModulePluginTestCase.java} (99%) rename modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/{GeoBoundsIT.java => GeoBoundsITTestCase.java} (99%) create mode 100644 modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidITTestCase.java rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java (87%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/composite/GeoTileValuesSource.java (88%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/BoundedCellValues.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/BucketPriorityQueue.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/CellIdSource.java (98%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/CellValues.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoGrid.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java (99%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoGridAggregator.java (99%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java (98%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java (95%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/InternalGeoGrid.java (99%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java (98%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java (94%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/ParsedGeoGrid.java (97%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java (93%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/UnboundedCellValues.java (96%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/package-info.java (79%) rename {server/src/main/java/org/opensearch => modules/geo/src/main/java/org/opensearch/geo}/search/aggregations/metrics/GeoGridAggregatorSupplier.java (93%) rename server/src/test/java/org/opensearch/search/aggregations/bucket/GeoHashGridTests.java => modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridAggregationBuilderTests.java (72%) rename server/src/test/java/org/opensearch/search/aggregations/bucket/GeoTileGridTests.java => modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridAggregationBuilderTests.java (70%) create mode 100644 modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridAggregationCompositeAggregatorTests.java create mode 100644 modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridCompositeAggregationBuilderTests.java rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java (90%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java (95%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoGridTestCase.java (79%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java (96%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java (99%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoHashGridTests.java (97%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java (94%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java (97%) rename {server/src/test/java/org/opensearch => modules/geo/src/test/java/org/opensearch/geo}/search/aggregations/bucket/geogrid/GeoTileGridTests.java (94%) create mode 100644 modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/230_composite.yml rename {rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation => modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape}/280_geohash_grid.yml (100%) rename {rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation => modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape}/290_geotile_grid.yml (100%) rename server/src/main/java/org/opensearch/search/aggregations/bucket/{geogrid => }/GeoTileUtils.java (97%) create mode 100644 server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationParsingFunction.java create mode 100644 test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java index 7ae8f8826c5a4..28a441bdf7f7f 100644 --- a/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/opensearch/client/RestHighLevelClient.java @@ -108,10 +108,6 @@ import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.opensearch.search.aggregations.bucket.filter.ParsedFilter; import org.opensearch.search.aggregations.bucket.filter.ParsedFilters; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; -import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.global.ParsedGlobal; import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -2130,8 +2126,6 @@ static List getDefaultNamedXContents() { map.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c)); map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); - map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); - map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c)); map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c)); diff --git a/modules/geo/build.gradle b/modules/geo/build.gradle index 0b8e623c24ac6..7f687a414e566 100644 --- a/modules/geo/build.gradle +++ b/modules/geo/build.gradle @@ -37,9 +37,16 @@ opensearchplugin { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + includeCore '_common', 'indices', 'index', 'search', 'bulk' } } artifacts { restTests(project.file('src/yamlRestTest/resources/rest-api-spec/test')) } +/** + * These compiler arguments needs to be removed, as there are raw types being used in the GeoGrid and GeoTile aggregations. + */ +tasks.withType(JavaCompile).configureEach { + options.compilerArgs -= '-Xlint:rawtypes' + options.compilerArgs -= '-Xlint:unchecked' +} diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoHashGridIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java similarity index 89% rename from server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoHashGridIT.java rename to modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java index 56d918feef9d8..6ab7dd5254679 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/GeoHashGridIT.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridIT.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket; +package org.opensearch.geo.search.aggregations.bucket; import com.carrotsearch.hppc.ObjectIntHashMap; import com.carrotsearch.hppc.ObjectIntMap; @@ -41,12 +41,12 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.settings.Settings; import org.opensearch.common.xcontent.XContentBuilder; +import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; +import org.opensearch.geo.tests.common.AggregationBuilders; import org.opensearch.index.query.GeoBoundingBoxQueryBuilder; -import org.opensearch.search.aggregations.AggregationBuilders; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.bucket.filter.Filter; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGrid; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGrid.Bucket; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.VersionUtils; @@ -57,17 +57,16 @@ import java.util.Random; import java.util.Set; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.geometry.utils.Geohash.PRECISION; import static org.opensearch.geometry.utils.Geohash.stringEncode; -import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.opensearch.search.aggregations.AggregationBuilders.geohashGrid; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class GeoHashGridIT extends OpenSearchIntegTestCase { +public class GeoHashGridIT extends GeoModulePluginIntegTestCase { @Override protected boolean forbidPrivateIndexSettings() { @@ -158,13 +157,13 @@ public void setupSuiteScopeCluster() throws Exception { public void testSimple() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) .get(); assertSearchResponse(response); GeoGrid geoGrid = response.getAggregations().get("geohashgrid"); - List buckets = geoGrid.getBuckets(); + List buckets = geoGrid.getBuckets(); Object[] propertiesKeys = (Object[]) ((InternalAggregation) geoGrid).getProperty("_key"); Object[] propertiesDocCounts = (Object[]) ((InternalAggregation) geoGrid).getProperty("_count"); for (int i = 0; i < buckets.size(); i++) { @@ -185,7 +184,7 @@ public void testSimple() throws Exception { public void testMultivalued() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("multi_valued_idx") - .addAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) .get(); assertSearchResponse(response); @@ -208,8 +207,8 @@ public void testFiltered() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") .addAggregation( - AggregationBuilders.filter("filtered", bbox) - .subAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + org.opensearch.search.aggregations.AggregationBuilders.filter("filtered", bbox) + .subAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) ) .get(); @@ -233,7 +232,7 @@ public void testFiltered() throws Exception { public void testUnmapped() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx_unmapped") - .addAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) .get(); assertSearchResponse(response); @@ -247,7 +246,7 @@ public void testUnmapped() throws Exception { public void testPartiallyUnmapped() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx", "idx_unmapped") - .addAggregation(geohashGrid("geohashgrid").field("location").precision(precision)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").precision(precision)) .get(); assertSearchResponse(response); @@ -267,7 +266,9 @@ public void testPartiallyUnmapped() throws Exception { public void testTopMatch() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid").field("location").size(1).shardSize(100).precision(precision)) + .addAggregation( + AggregationBuilders.geohashGrid("geohashgrid").field("location").size(1).shardSize(100).precision(precision) + ) .get(); assertSearchResponse(response); @@ -296,7 +297,7 @@ public void testSizeIsZero() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) .get() ); assertThat(exception.getMessage(), containsString("[size] must be greater than 0. Found [0] in [geohashgrid]")); @@ -308,7 +309,7 @@ public void testShardSizeIsZero() { IllegalArgumentException exception = expectThrows( IllegalArgumentException.class, () -> client().prepareSearch("idx") - .addAggregation(geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) + .addAggregation(AggregationBuilders.geohashGrid("geohashgrid").field("location").size(size).shardSize(shardSize)) .get() ); assertThat(exception.getMessage(), containsString("[shardSize] must be greater than 0. Found [0] in [geohashgrid]")); diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java new file mode 100644 index 0000000000000..5b4dd052a2f65 --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/bucket/ShardReduceIT.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket; + +import org.opensearch.action.index.IndexRequestBuilder; +import org.opensearch.action.search.SearchResponse; +import org.opensearch.geo.GeoModulePluginIntegTestCase; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; +import org.opensearch.geo.tests.common.AggregationBuilders; +import org.opensearch.geometry.utils.Geohash; +import org.opensearch.index.query.QueryBuilders; +import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; +import org.opensearch.search.aggregations.bucket.histogram.Histogram; +import org.opensearch.test.OpenSearchIntegTestCase; + +import static org.hamcrest.Matchers.equalTo; +import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; + +/** + * Tests making sure that the reduce is propagated to all aggregations in the hierarchy when executing on a single shard + * These tests are based on the date histogram in combination of min_doc_count=0. In order for the date histogram to + * compute empty buckets, its {@code reduce()} method must be called. So by adding the date histogram under other buckets, + * we can make sure that the reduce is properly propagated by checking that empty buckets were created. + */ +@OpenSearchIntegTestCase.SuiteScopeTestCase +public class ShardReduceIT extends GeoModulePluginIntegTestCase { + + private IndexRequestBuilder indexDoc(String date, int value) throws Exception { + return client().prepareIndex("idx") + .setSource( + jsonBuilder().startObject() + .field("value", value) + .field("ip", "10.0.0." + value) + .field("location", Geohash.stringEncode(5, 52, Geohash.PRECISION)) + .field("date", date) + .field("term-l", 1) + .field("term-d", 1.5) + .field("term-s", "term") + .startObject("nested") + .field("date", date) + .endObject() + .endObject() + ); + } + + @Override + public void setupSuiteScopeCluster() throws Exception { + assertAcked( + prepareCreate("idx").setMapping( + "nested", + "type=nested", + "ip", + "type=ip", + "location", + "type=geo_point", + "term-s", + "type=keyword" + ) + ); + + indexRandom(true, indexDoc("2014-01-01", 1), indexDoc("2014-01-02", 2), indexDoc("2014-01-04", 3)); + ensureSearchable(); + } + + public void testGeoHashGrid() throws Exception { + SearchResponse response = client().prepareSearch("idx") + .setQuery(QueryBuilders.matchAllQuery()) + .addAggregation( + AggregationBuilders.geohashGrid("grid") + .field("location") + .subAggregation(dateHistogram("histo").field("date").fixedInterval(DateHistogramInterval.DAY).minDocCount(0)) + ) + .get(); + + assertSearchResponse(response); + + GeoGrid grid = response.getAggregations().get("grid"); + Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); + assertThat(histo.getBuckets().size(), equalTo(4)); + } + + public void testGeoTileGrid() throws Exception { + SearchResponse response = client().prepareSearch("idx") + .setQuery(QueryBuilders.matchAllQuery()) + .addAggregation( + AggregationBuilders.geotileGrid("grid") + .field("location") + .subAggregation(dateHistogram("histo").field("date").fixedInterval(DateHistogramInterval.DAY).minDocCount(0)) + ) + .get(); + + assertSearchResponse(response); + + GeoGrid grid = response.getAggregations().get("grid"); + Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); + assertThat(histo.getBuckets().size(), equalTo(4)); + } +} diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorTestCaseModulePlugin.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java similarity index 99% rename from modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorTestCaseModulePlugin.java rename to modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java index 0065cca7d6101..92987d407f51d 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorTestCaseModulePlugin.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/AbstractGeoAggregatorModulePluginTestCase.java @@ -42,7 +42,7 @@ * to copy the code as we cannot depend on this class. * GitHub issue */ -public abstract class AbstractGeoAggregatorTestCaseModulePlugin extends GeoModulePluginIntegTestCase { +public abstract class AbstractGeoAggregatorModulePluginTestCase extends GeoModulePluginIntegTestCase { protected static final String SINGLE_VALUED_FIELD_NAME = "geo_value"; protected static final String MULTI_VALUED_FIELD_NAME = "geo_values"; diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsIT.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java similarity index 99% rename from modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsIT.java rename to modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java index 5cbd98a4936e4..8cc82da12d69a 100644 --- a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsIT.java +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoBoundsITTestCase.java @@ -57,7 +57,7 @@ import static org.opensearch.geo.tests.common.AggregationBuilders.geoBounds; @OpenSearchIntegTestCase.SuiteScopeTestCase -public class GeoBoundsIT extends AbstractGeoAggregatorTestCaseModulePlugin { +public class GeoBoundsITTestCase extends AbstractGeoAggregatorModulePluginTestCase { private static final String aggName = "geoBounds"; public void testSingleValuedField() throws Exception { diff --git a/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidITTestCase.java b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidITTestCase.java new file mode 100644 index 0000000000000..e6d45e27b8f70 --- /dev/null +++ b/modules/geo/src/internalClusterTest/java/org/opensearch/geo/search/aggregations/metrics/GeoCentroidITTestCase.java @@ -0,0 +1,84 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.geo.search.aggregations.metrics; + +import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGrid; +import org.opensearch.geo.tests.common.AggregationBuilders; +import org.opensearch.search.aggregations.metrics.GeoCentroid; +import org.opensearch.test.OpenSearchIntegTestCase; + +import java.util.List; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.opensearch.search.aggregations.AggregationBuilders.geoCentroid; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; + +@OpenSearchIntegTestCase.SuiteScopeTestCase +public class GeoCentroidITTestCase extends AbstractGeoAggregatorModulePluginTestCase { + private static final String aggName = "geoCentroid"; + + public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception { + SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) + .addAggregation( + AggregationBuilders.geohashGrid("geoGrid") + .field(SINGLE_VALUED_FIELD_NAME) + .subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + ) + .get(); + assertSearchResponse(response); + + GeoGrid grid = response.getAggregations().get("geoGrid"); + assertThat(grid, notNullValue()); + assertThat(grid.getName(), equalTo("geoGrid")); + List buckets = grid.getBuckets(); + for (GeoGrid.Bucket cell : buckets) { + String geohash = cell.getKeyAsString(); + GeoPoint expectedCentroid = expectedCentroidsForGeoHash.get(geohash); + GeoCentroid centroidAgg = cell.getAggregations().get(aggName); + assertThat( + "Geohash " + geohash + " has wrong centroid latitude ", + expectedCentroid.lat(), + closeTo(centroidAgg.centroid().lat(), GEOHASH_TOLERANCE) + ); + assertThat( + "Geohash " + geohash + " has wrong centroid longitude", + expectedCentroid.lon(), + closeTo(centroidAgg.centroid().lon(), GEOHASH_TOLERANCE) + ); + } + } +} diff --git a/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java b/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java index 64aac66b7eef3..25dcf8db2c407 100644 --- a/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java +++ b/modules/geo/src/main/java/org/opensearch/geo/GeoModulePlugin.java @@ -32,6 +32,12 @@ package org.opensearch.geo; +import org.opensearch.geo.search.aggregations.bucket.composite.GeoTileGridValuesSourceBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregator; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoHashGrid; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.opensearch.geo.search.aggregations.metrics.GeoBounds; import org.opensearch.geo.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.opensearch.geo.search.aggregations.metrics.InternalGeoBounds; @@ -40,6 +46,7 @@ import org.opensearch.plugins.MapperPlugin; import org.opensearch.plugins.Plugin; import org.opensearch.plugins.SearchPlugin; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; import java.util.Collections; import java.util.List; @@ -57,11 +64,42 @@ public Map getMappers() { */ @Override public List getAggregations() { - final AggregationSpec spec = new AggregationSpec( + final AggregationSpec geoBounds = new AggregationSpec( GeoBoundsAggregationBuilder.NAME, GeoBoundsAggregationBuilder::new, GeoBoundsAggregationBuilder.PARSER ).addResultReader(InternalGeoBounds::new).setAggregatorRegistrar(GeoBoundsAggregationBuilder::registerAggregators); - return Collections.singletonList(spec); + + final AggregationSpec geoHashGrid = new AggregationSpec( + GeoHashGridAggregationBuilder.NAME, + GeoHashGridAggregationBuilder::new, + GeoHashGridAggregationBuilder.PARSER + ).addResultReader(InternalGeoHashGrid::new).setAggregatorRegistrar(GeoHashGridAggregationBuilder::registerAggregators); + + final AggregationSpec geoTileGrid = new AggregationSpec( + GeoTileGridAggregationBuilder.NAME, + GeoTileGridAggregationBuilder::new, + GeoTileGridAggregationBuilder.PARSER + ).addResultReader(InternalGeoTileGrid::new).setAggregatorRegistrar(GeoTileGridAggregationBuilder::registerAggregators); + return List.of(geoBounds, geoHashGrid, geoTileGrid); + } + + /** + * Registering the {@link GeoTileGridAggregator} in the {@link CompositeAggregation}. + * + * @return a {@link List} of {@link CompositeAggregationSpec} + */ + @Override + public List getCompositeAggregations() { + return Collections.singletonList( + new CompositeAggregationSpec( + GeoTileGridValuesSourceBuilder::register, + GeoTileGridValuesSourceBuilder.class, + GeoTileGridValuesSourceBuilder.COMPOSITE_AGGREGATION_SERIALISATION_BYTE_CODE, + GeoTileGridValuesSourceBuilder::new, + GeoTileGridValuesSourceBuilder::parse, + GeoTileGridValuesSourceBuilder.TYPE + ) + ); } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java similarity index 87% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java index 4b01a08d29a43..84d5943da287f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.composite; +package org.opensearch.geo.search.aggregations.bucket.composite; import org.apache.lucene.index.IndexReader; import org.opensearch.LegacyESVersion; @@ -43,12 +43,15 @@ import org.opensearch.common.xcontent.ObjectParser; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.geo.search.aggregations.bucket.geogrid.CellIdSource; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.query.QueryShardContext; import org.opensearch.search.DocValueFormat; -import org.opensearch.search.aggregations.bucket.geogrid.CellIdSource; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceConfig; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceParserHelper; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.aggregations.support.ValuesSource; @@ -88,13 +91,19 @@ CompositeValuesSourceConfig apply( ); } - static final String TYPE = "geotile_grid"; + public static final String TYPE = "geotile_grid"; + /* + use the TYPE parameter instead of Byte code. The byte code is added for backward compatibility and will be + removed in the next version. + */ + @Deprecated + public static final Byte COMPOSITE_AGGREGATION_SERIALISATION_BYTE_CODE = 3; static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = new ValuesSourceRegistry.RegistryKey( TYPE, GeoTileCompositeSuppier.class ); - private static final ObjectParser PARSER; + static final ObjectParser PARSER; static { PARSER = new ObjectParser<>(GeoTileGridValuesSourceBuilder.TYPE); PARSER.declareInt(GeoTileGridValuesSourceBuilder::precision, new ParseField("precision")); @@ -106,11 +115,11 @@ CompositeValuesSourceConfig apply( CompositeValuesSourceParserHelper.declareValuesSourceFields(PARSER); } - static GeoTileGridValuesSourceBuilder parse(String name, XContentParser parser) throws IOException { + public static GeoTileGridValuesSourceBuilder parse(String name, XContentParser parser) throws IOException { return PARSER.parse(parser, new GeoTileGridValuesSourceBuilder(name), null); } - static void register(ValuesSourceRegistry.Builder builder) { + public static void register(ValuesSourceRegistry.Builder builder) { builder.register( REGISTRY_KEY, @@ -163,7 +172,7 @@ static void register(ValuesSourceRegistry.Builder builder) { super(name); } - GeoTileGridValuesSourceBuilder(StreamInput in) throws IOException { + public GeoTileGridValuesSourceBuilder(StreamInput in) throws IOException { super(in); this.precision = in.readInt(); if (in.getVersion().onOrAfter(LegacyESVersion.V_7_6_0)) { @@ -203,7 +212,7 @@ protected void doXContentBody(XContentBuilder builder, Params params) throws IOE } @Override - String type() { + protected String type() { return TYPE; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileValuesSource.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java similarity index 88% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileValuesSource.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java index 819dfc573bbe4..303e577e99e7b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/GeoTileValuesSource.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileValuesSource.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.composite; +package org.opensearch.geo.search.aggregations.bucket.composite; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; @@ -38,7 +38,9 @@ import org.opensearch.common.util.BigArrays; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.search.DocValueFormat; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.composite.LongValuesSource; +import org.opensearch.search.aggregations.bucket.composite.SingleDimensionValuesSource; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import java.io.IOException; @@ -68,7 +70,7 @@ class GeoTileValuesSource extends LongValuesSource { } @Override - void setAfter(Comparable value) { + protected void setAfter(Comparable value) { if (missingBucket && value == null) { afterValue = null; } else if (value instanceof Number) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BoundedCellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BoundedCellValues.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BoundedCellValues.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BoundedCellValues.java index ba824fc8f21dd..06d2dcaee3932 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BoundedCellValues.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BoundedCellValues.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.index.fielddata.MultiGeoPointValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BucketPriorityQueue.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BucketPriorityQueue.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BucketPriorityQueue.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BucketPriorityQueue.java index d6cfde0c46eae..70d0552b3e80b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/BucketPriorityQueue.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/BucketPriorityQueue.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.util.PriorityQueue; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellIdSource.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellIdSource.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellIdSource.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellIdSource.java index 12d9043a2fd5f..d40029e9a762d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellIdSource.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellIdSource.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellValues.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellValues.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellValues.java index 9dc357659aae8..d01896c8136fa 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/CellValues.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/CellValues.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.index.fielddata.AbstractSortingNumericDocValues; import org.opensearch.index.fielddata.MultiGeoPointValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGrid.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGrid.java index cfdb08f9ee3d7..4ae888640efc8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGrid.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java similarity index 99% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index b08c40268c5cf..4a904b3aa2b16 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.LegacyESVersion; import org.opensearch.OpenSearchException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregator.java similarity index 99% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregator.java index 1ef8ba6c697f4..909772c61a960 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregator.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregator.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java index 4049bf2c73640..bbaf9613fb216 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoUtils; @@ -40,7 +40,7 @@ import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; -import org.opensearch.search.aggregations.metrics.GeoGridAggregatorSupplier; +import org.opensearch.geo.search.aggregations.metrics.GeoGridAggregatorSupplier; import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java index 1106320c7431f..6ca7a4d8a9cb8 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregator.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java index cdc801aaedffb..1914c07e831f7 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.geometry.utils.Geohash; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java similarity index 95% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java index f73360e3cb826..76ad515f34fe5 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.io.stream.StreamInput; @@ -39,7 +39,8 @@ import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.AggregatorFactory; -import org.opensearch.search.aggregations.metrics.GeoGridAggregatorSupplier; +import org.opensearch.geo.search.aggregations.metrics.GeoGridAggregatorSupplier; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.aggregations.support.ValuesSourceRegistry; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java index 7a2b908148c4c..a205a9afde41e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregator.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java index ef8cd11a22498..b830988a3d410 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.index.query.QueryShardContext; @@ -40,6 +40,7 @@ import org.opensearch.search.aggregations.CardinalityUpperBound; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.NonCollectingAggregator; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGrid.java similarity index 99% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGrid.java index 94a5ad5717854..9dbed7b27307a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGrid.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java similarity index 98% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java index a187bfefb661f..93fcdbd098400 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoGridBucket.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java index 7811b8774d04f..ff1247300939a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGrid.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java index f9c45dc41ceb1..659909e868651 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoHashGridBucket.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.io.stream.StreamInput; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java index efbd9a05d6a4d..fa544b5893f0c 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGrid.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java similarity index 94% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java index f200f55232e00..65d736cfceb32 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/InternalGeoTileGridBucket.java @@ -30,11 +30,12 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.search.aggregations.InternalAggregations; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGrid.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGrid.java index 3f85cf350c89c..adfffeddba59d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGrid.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.CheckedFunction; import org.opensearch.common.xcontent.ObjectParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java index 08e5c15188ee6..80124cda50b19 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoGridBucket.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.search.aggregations.Aggregation; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java index f20f972c1ce0a..109524e755c4d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGrid.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.xcontent.ObjectParser; import org.opensearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java index 05c7a1c8d1663..4e6e454b08324 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoHashGridBucket.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java index 06915cc4210e1..8734c96a15578 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGrid.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.xcontent.ObjectParser; import org.opensearch.common.xcontent.XContentParser; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java similarity index 93% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java index c8dec16f322ef..fd47c35f13de1 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/ParsedGeoTileGridBucket.java @@ -30,10 +30,11 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoPoint; import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import java.io.IOException; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/UnboundedCellValues.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/UnboundedCellValues.java similarity index 96% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/UnboundedCellValues.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/UnboundedCellValues.java index f5a139cdb8d9d..c628c7bfdc8ec 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/UnboundedCellValues.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/UnboundedCellValues.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.index.fielddata.MultiGeoPointValues; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/package-info.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/package-info.java similarity index 79% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/package-info.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/package-info.java index c59685e06cf79..d9183a0f742ef 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/package-info.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/bucket/geogrid/package-info.java @@ -7,4 +7,4 @@ */ /** geo_grid Aggregation package. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoGridAggregatorSupplier.java similarity index 93% rename from server/src/main/java/org/opensearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java rename to modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoGridAggregatorSupplier.java index 183c64f4e4af2..43ccb8b89545a 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/GeoGridAggregatorSupplier.java +++ b/modules/geo/src/main/java/org/opensearch/geo/search/aggregations/metrics/GeoGridAggregatorSupplier.java @@ -30,13 +30,13 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.metrics; +package org.opensearch.geo.search.aggregations.metrics; import org.opensearch.common.geo.GeoBoundingBox; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregator; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.CardinalityUpperBound; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGridAggregator; import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.internal.SearchContext; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/GeoHashGridTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridAggregationBuilderTests.java similarity index 72% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/GeoHashGridTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridAggregationBuilderTests.java index 5e230a445ec98..00cb162e64c19 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/GeoHashGridTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoHashGridAggregationBuilderTests.java @@ -30,14 +30,23 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket; +package org.opensearch.geo.search.aggregations.bucket; -import org.opensearch.common.geo.GeoBoundingBoxTests; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.plugins.Plugin; import org.opensearch.search.aggregations.BaseAggregationTestCase; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -public class GeoHashGridTests extends BaseAggregationTestCase { +import java.util.Collection; +import java.util.Collections; + +public class GeoHashGridAggregationBuilderTests extends BaseAggregationTestCase { + + protected Collection> getPlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } @Override protected GeoHashGridAggregationBuilder createTestAggregatorBuilder() { @@ -55,7 +64,7 @@ protected GeoHashGridAggregationBuilder createTestAggregatorBuilder() { factory.shardSize(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { - factory.setGeoBoundingBox(GeoBoundingBoxTests.randomBBox()); + factory.setGeoBoundingBox(RandomGeoGenerator.randomBBox()); } return factory; } diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/GeoTileGridTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridAggregationBuilderTests.java similarity index 70% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/GeoTileGridTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridAggregationBuilderTests.java index d54667fb4f1a6..c7c0be21273bd 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/GeoTileGridTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/GeoTileGridAggregationBuilderTests.java @@ -30,15 +30,24 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket; +package org.opensearch.geo.search.aggregations.bucket; -import org.opensearch.common.geo.GeoBoundingBoxTests; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.plugins.Plugin; import org.opensearch.search.aggregations.BaseAggregationTestCase; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; -public class GeoTileGridTests extends BaseAggregationTestCase { +import java.util.Collection; +import java.util.Collections; + +public class GeoTileGridAggregationBuilderTests extends BaseAggregationTestCase { + + protected Collection> getPlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } @Override protected GeoTileGridAggregationBuilder createTestAggregatorBuilder() { @@ -55,7 +64,7 @@ protected GeoTileGridAggregationBuilder createTestAggregatorBuilder() { factory.shardSize(randomIntBetween(1, Integer.MAX_VALUE)); } if (randomBoolean()) { - factory.setGeoBoundingBox(GeoBoundingBoxTests.randomBBox()); + factory.setGeoBoundingBox(RandomGeoGenerator.randomBBox()); } return factory; } diff --git a/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridAggregationCompositeAggregatorTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridAggregationCompositeAggregatorTests.java new file mode 100644 index 0000000000000..3c7c292f9d193 --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridAggregationCompositeAggregatorTests.java @@ -0,0 +1,174 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket.composite; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LatLonPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.junit.Before; +import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregator; +import org.opensearch.index.mapper.GeoPointFieldMapper; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; +import org.opensearch.search.aggregations.composite.BaseCompositeAggregatorTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** + * Testing the {@link GeoTileGridAggregator} as part of CompositeAggregation. + */ +public class GeoTileGridAggregationCompositeAggregatorTests extends BaseCompositeAggregatorTestCase { + + protected List getSearchPlugins() { + return Collections.singletonList(new GeoModulePlugin()); + } + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + FIELD_TYPES.add(new GeoPointFieldMapper.GeoPointFieldType("geo_point")); + } + + public void testUnmappedFieldWithGeopoint() throws Exception { + final List>> dataset = new ArrayList<>(); + final String mappedFieldName = "geo_point"; + dataset.addAll( + Arrays.asList( + createDocument(mappedFieldName, new GeoPoint(48.934059, 41.610741)), + createDocument(mappedFieldName, new GeoPoint(-23.065941, 113.610741)), + createDocument(mappedFieldName, new GeoPoint(90.0, 0.0)), + createDocument(mappedFieldName, new GeoPoint(37.2343, -115.8067)), + createDocument(mappedFieldName, new GeoPoint(90.0, 0.0)) + ) + ); + + // just unmapped = no results + testSearchCase( + Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), + dataset, + () -> new CompositeAggregationBuilder("name", Arrays.asList(new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped"))), + (result) -> assertEquals(0, result.getBuckets().size()) + ); + + // unmapped missing bucket = one result + testSearchCase( + Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), + dataset, + () -> new CompositeAggregationBuilder( + "name", + Arrays.asList(new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped").missingBucket(true)) + ), + (result) -> { + assertEquals(1, result.getBuckets().size()); + assertEquals("{unmapped=null}", result.afterKey().toString()); + assertEquals("{unmapped=null}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(5L, result.getBuckets().get(0).getDocCount()); + } + ); + + // field + unmapped, no missing bucket = no results + testSearchCase( + Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), + dataset, + () -> new CompositeAggregationBuilder( + "name", + Arrays.asList( + new GeoTileGridValuesSourceBuilder(mappedFieldName).field(mappedFieldName), + new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped") + ) + ), + (result) -> assertEquals(0, result.getBuckets().size()) + ); + + // field + unmapped with missing bucket = multiple results + testSearchCase( + Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), + dataset, + () -> new CompositeAggregationBuilder( + "name", + Arrays.asList( + new GeoTileGridValuesSourceBuilder(mappedFieldName).field(mappedFieldName), + new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped").missingBucket(true) + ) + ), + (result) -> { + assertEquals(2, result.getBuckets().size()); + assertEquals("{geo_point=7/64/56, unmapped=null}", result.afterKey().toString()); + assertEquals("{geo_point=7/32/56, unmapped=null}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(0).getDocCount()); + assertEquals("{geo_point=7/64/56, unmapped=null}", result.getBuckets().get(1).getKeyAsString()); + assertEquals(3L, result.getBuckets().get(1).getDocCount()); + } + ); + + } + + public void testWithGeoPoint() throws Exception { + final List>> dataset = new ArrayList<>(); + dataset.addAll( + Arrays.asList( + createDocument("geo_point", new GeoPoint(48.934059, 41.610741)), + createDocument("geo_point", new GeoPoint(-23.065941, 113.610741)), + createDocument("geo_point", new GeoPoint(90.0, 0.0)), + createDocument("geo_point", new GeoPoint(37.2343, -115.8067)), + createDocument("geo_point", new GeoPoint(90.0, 0.0)) + ) + ); + testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset, () -> { + GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point").field("geo_point"); + return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile)); + }, (result) -> { + assertEquals(2, result.getBuckets().size()); + assertEquals("{geo_point=7/64/56}", result.afterKey().toString()); + assertEquals("{geo_point=7/32/56}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(2L, result.getBuckets().get(0).getDocCount()); + assertEquals("{geo_point=7/64/56}", result.getBuckets().get(1).getKeyAsString()); + assertEquals(3L, result.getBuckets().get(1).getDocCount()); + }); + + testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset, () -> { + GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point").field("geo_point"); + return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile)).aggregateAfter( + Collections.singletonMap("geo_point", "7/32/56") + ); + }, (result) -> { + assertEquals(1, result.getBuckets().size()); + assertEquals("{geo_point=7/64/56}", result.afterKey().toString()); + assertEquals("{geo_point=7/64/56}", result.getBuckets().get(0).getKeyAsString()); + assertEquals(3L, result.getBuckets().get(0).getDocCount()); + }); + } + + @Override + protected boolean addValueToDocument(final Document doc, final String name, final Object value) { + if (value instanceof GeoPoint) { + GeoPoint point = (GeoPoint) value; + doc.add( + new SortedNumericDocValuesField( + name, + GeoTileUtils.longEncode(point.lon(), point.lat(), GeoTileGridAggregationBuilder.DEFAULT_PRECISION) + ) + ); + doc.add(new LatLonPoint(name, point.lat(), point.lon())); + return true; + } + return false; + } +} diff --git a/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridCompositeAggregationBuilderTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridCompositeAggregationBuilderTests.java new file mode 100644 index 0000000000000..ea7a2a83945c2 --- /dev/null +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridCompositeAggregationBuilderTests.java @@ -0,0 +1,50 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.geo.search.aggregations.bucket.composite; + +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.tests.common.RandomGeoGenerator; +import org.opensearch.plugins.Plugin; +import org.opensearch.search.aggregations.BaseAggregationTestCase; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +public class GeoTileGridCompositeAggregationBuilderTests extends BaseAggregationTestCase { + + protected Collection> getPlugins() { + return Collections.singletonList(GeoModulePlugin.class); + } + + private GeoTileGridValuesSourceBuilder randomGeoTileGridValuesSourceBuilder() { + GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10)); + if (randomBoolean()) { + geoTile.precision(randomIntBetween(0, GeoTileUtils.MAX_ZOOM)); + } + if (randomBoolean()) { + geoTile.geoBoundingBox(RandomGeoGenerator.randomBBox()); + } + return geoTile; + } + + @Override + protected CompositeAggregationBuilder createTestAggregatorBuilder() { + int numSources = randomIntBetween(1, 10); + List> sources = new ArrayList<>(); + for (int i = 0; i < numSources; i++) { + sources.add(randomGeoTileGridValuesSourceBuilder()); + } + return new CompositeAggregationBuilder(randomAlphaOfLength(10), sources); + } +} diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java similarity index 90% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java index 2b1700676f549..c6276c06c4511 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilderTests.java @@ -30,8 +30,9 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.composite; +package org.opensearch.geo.search.aggregations.bucket.composite; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.opensearch.test.OpenSearchTestCase; public class GeoTileGridValuesSourceBuilderTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java similarity index 95% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java index 17fddb8978499..d6153637f656d 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; @@ -45,17 +45,19 @@ import org.apache.lucene.util.BytesRef; import org.opensearch.common.CheckedConsumer; import org.opensearch.common.geo.GeoBoundingBox; -import org.opensearch.common.geo.GeoBoundingBoxTests; import org.opensearch.common.geo.GeoUtils; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.tests.common.AggregationInspectionHelper; +import org.opensearch.geo.tests.common.RandomGeoGenerator; import org.opensearch.index.mapper.GeoPointFieldMapper; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.plugins.SearchPlugin; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorTestCase; import org.opensearch.search.aggregations.MultiBucketConsumerService; import org.opensearch.search.aggregations.bucket.terms.StringTerms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.opensearch.search.aggregations.support.AggregationInspectionHelper; import java.io.IOException; import java.util.ArrayList; @@ -91,6 +93,16 @@ public abstract class GeoGridAggregatorTestCase */ protected abstract GeoGridAggregationBuilder createBuilder(String name); + /** + * Overriding the Search Plugins list with {@link GeoModulePlugin} so that the testcase will know that this plugin is + * to be loaded during the tests. + * @return List of {@link SearchPlugin} + */ + @Override + protected List getSearchPlugins() { + return Collections.singletonList(new GeoModulePlugin()); + } + public void testNoDocs() throws IOException { testCase( new MatchAllDocsQuery(), @@ -225,7 +237,7 @@ public void testBounds() throws IOException { // only consider bounding boxes that are at least GEOHASH_TOLERANCE wide and have quantized coordinates GeoBoundingBox bbox = randomValueOtherThanMany( (b) -> Math.abs(GeoUtils.normalizeLon(b.right()) - GeoUtils.normalizeLon(b.left())) < GEOHASH_TOLERANCE, - GeoBoundingBoxTests::randomBBox + RandomGeoGenerator::randomBBox ); Function encodeDecodeLat = (lat) -> GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); Function encodeDecodeLon = (lon) -> GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon)); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridTestCase.java similarity index 79% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridTestCase.java index ce286a4443660..432736a2b43fe 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoGridTestCase.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoGridTestCase.java @@ -29,9 +29,16 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.apache.lucene.index.IndexWriter; +import org.opensearch.common.ParseField; +import org.opensearch.common.xcontent.ContextParser; +import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.geo.GeoModulePlugin; +import org.opensearch.geo.search.aggregations.metrics.ParsedGeoBounds; +import org.opensearch.plugins.SearchPlugin; +import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.InternalAggregations; import org.opensearch.search.aggregations.ParsedMultiBucketAggregation; import org.opensearch.test.InternalMultiBucketAggregationTestCase; @@ -76,6 +83,36 @@ protected int maxNumberOfBuckets() { return 3; } + /** + * Overriding the method so that tests can get the aggregation specs for namedWriteable. + * + * @return GeoPlugin + */ + @Override + protected SearchPlugin registerPlugin() { + return new GeoModulePlugin(); + } + + /** + * Overriding with the {@link ParsedGeoBounds} so that it can be parsed. We need to do this as {@link GeoModulePlugin} + * is registering this Aggregation. + * + * @return a List of {@link NamedXContentRegistry.Entry} + */ + @Override + protected List getNamedXContents() { + final List namedXContents = new ArrayList<>(getDefaultNamedXContents()); + final ContextParser hashGridParser = (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c); + final ContextParser geoTileParser = (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c); + namedXContents.add( + new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(GeoHashGridAggregationBuilder.NAME), hashGridParser) + ); + namedXContents.add( + new NamedXContentRegistry.Entry(Aggregation.class, new ParseField(GeoTileGridAggregationBuilder.NAME), geoTileParser) + ); + return namedXContents; + } + @Override protected T createTestInstance(String name, Map metadata, InternalAggregations aggregations) { final int precision = randomPrecision(); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java similarity index 96% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java index 5c63b15c7f614..04fa815366f6b 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java @@ -30,7 +30,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import static org.opensearch.geometry.utils.Geohash.stringEncode; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java similarity index 99% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java index e81e22b3b562f..44f292e898a61 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridParserTests.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.greaterThanOrEqualTo; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridTests.java similarity index 97% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridTests.java index 5a26ec759281c..c84c6ef5ec076 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoHashGridTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoHashGridTests.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.geometry.utils.Geohash; import org.opensearch.search.aggregations.InternalAggregations; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java similarity index 94% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java index 4e88111ac2dfc..f2f641ea794c0 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java @@ -30,7 +30,9 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; + +import org.opensearch.search.aggregations.bucket.GeoTileUtils; public class GeoTileGridAggregatorTests extends GeoGridAggregatorTestCase { diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java similarity index 97% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java index 567bcd57d23e5..a5b000d5e6ab3 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridParserTests.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.ExceptionsHelper; import org.opensearch.common.xcontent.XContentParseException; @@ -37,6 +37,7 @@ import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.geo.GeometryTestUtils; import org.opensearch.geometry.Rectangle; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.test.OpenSearchTestCase; import static org.hamcrest.Matchers.containsString; diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridTests.java similarity index 94% rename from server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java rename to modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridTests.java index 50b9a8cd762d1..ead67e0455d94 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileGridTests.java +++ b/modules/geo/src/test/java/org/opensearch/geo/search/aggregations/bucket/geogrid/GeoTileGridTests.java @@ -29,9 +29,10 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.geo.search.aggregations.bucket.geogrid; import org.opensearch.search.aggregations.InternalAggregations; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import java.util.List; import java.util.Map; diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java index c1f27b71c326d..c0d7e51047c6b 100644 --- a/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationBuilders.java @@ -8,6 +8,10 @@ package org.opensearch.geo.tests.common; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoHashGrid; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.opensearch.geo.search.aggregations.metrics.GeoBounds; import org.opensearch.geo.search.aggregations.metrics.GeoBoundsAggregationBuilder; @@ -18,4 +22,18 @@ public class AggregationBuilders { public static GeoBoundsAggregationBuilder geoBounds(String name) { return new GeoBoundsAggregationBuilder(name); } + + /** + * Create a new {@link InternalGeoHashGrid} aggregation with the given name. + */ + public static GeoHashGridAggregationBuilder geohashGrid(String name) { + return new GeoHashGridAggregationBuilder(name); + } + + /** + * Create a new {@link InternalGeoTileGrid} aggregation with the given name. + */ + public static GeoTileGridAggregationBuilder geotileGrid(String name) { + return new GeoTileGridAggregationBuilder(name); + } } diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java index 208187bf34a5c..3473cf2d94b76 100644 --- a/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/AggregationInspectionHelper.java @@ -8,6 +8,7 @@ package org.opensearch.geo.tests.common; +import org.opensearch.geo.search.aggregations.bucket.geogrid.InternalGeoGrid; import org.opensearch.geo.search.aggregations.metrics.InternalGeoBounds; public class AggregationInspectionHelper { @@ -15,4 +16,8 @@ public class AggregationInspectionHelper { public static boolean hasValue(InternalGeoBounds agg) { return (agg.topLeft() == null && agg.bottomRight() == null) == false; } + + public static boolean hasValue(InternalGeoGrid agg) { + return agg.getBuckets().stream().anyMatch(bucket -> bucket.getDocCount() > 0); + } } diff --git a/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java index 2cf32c36b97ec..2fb403155e2bc 100644 --- a/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java +++ b/modules/geo/src/test/java/org/opensearch/geo/tests/common/RandomGeoGenerator.java @@ -8,7 +8,10 @@ package org.opensearch.geo.tests.common; +import org.opensearch.common.geo.GeoBoundingBox; import org.opensearch.common.geo.GeoPoint; +import org.opensearch.geo.GeometryTestUtils; +import org.opensearch.geometry.Rectangle; import java.util.Random; @@ -83,4 +86,12 @@ private static double normalizeLongitude(double longitude) { return -180 + off; } } + + public static GeoBoundingBox randomBBox() { + Rectangle rectangle = GeometryTestUtils.randomRectangle(); + return new GeoBoundingBox( + new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) + ); + } } diff --git a/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/230_composite.yml b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/230_composite.yml new file mode 100644 index 0000000000000..211f3c3f46b88 --- /dev/null +++ b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/230_composite.yml @@ -0,0 +1,168 @@ +--- +setup: + - do: + indices.create: + index: test + body: + mappings: + properties: + date: + type: date + keyword: + type: keyword + long: + type: long + geo_point: + type: geo_point + nested: + type: nested + properties: + nested_long: + type: long + + - do: + indices.create: + index: other + body: + mappings: + properties: + date: + type: date + long: + type: long + nested: + type: nested + properties: + nested_long: + type: long + + - do: + index: + index: test + id: 1 + body: { "keyword": "foo", "long": [10, 20], "geo_point": "37.2343,-115.8067", "nested": [{"nested_long": 10}, {"nested_long": 20}] } + + - do: + index: + index: test + id: 2 + body: { "keyword": ["foo", "bar"], "geo_point": "41.12,-71.34" } + + - do: + index: + index: test + id: 3 + body: { "keyword": "bar", "long": [100, 0], "geo_point": "90.0,0.0", "nested": [{"nested_long": 10}, {"nested_long": 0}] } + + - do: + index: + index: test + id: 4 + body: { "keyword": "bar", "long": [1000, 0], "geo_point": "41.12,-71.34", "nested": [{"nested_long": 1000}, {"nested_long": 20}] } + + - do: + index: + index: test + id: 5 + body: { "date": "2017-10-20T03:08:45" } + + - do: + index: + index: test + id: 6 + body: { "date": "2017-10-21T07:00:00" } + + - do: + index: + index: other + id: 0 + body: { "date": "2017-10-20T03:08:45" } + + - do: + indices.refresh: + index: [test, other] +--- +"Simple Composite aggregation with GeoTile grid": + - skip: + version: " - 7.4.99" + reason: geotile_grid is not supported until 7.5.0 + - do: + search: + rest_total_hits_as_int: true + index: test + body: + aggregations: + test: + composite: + sources: [ + "geo": { + "geotile_grid": { + "field": "geo_point", + "precision": 12 + } + }, + { + "kw": { + "terms": { + "field": "keyword" + } + } + } + ] + + - match: {hits.total: 6} + - length: { aggregations.test.buckets: 4 } + - match: { aggregations.test.buckets.0.key.geo: "12/730/1590" } + - match: { aggregations.test.buckets.0.key.kw: "foo" } + - match: { aggregations.test.buckets.0.doc_count: 1 } + - match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" } + - match: { aggregations.test.buckets.1.key.kw: "bar" } + - match: { aggregations.test.buckets.1.doc_count: 2 } + - match: { aggregations.test.buckets.2.key.geo: "12/1236/1533" } + - match: { aggregations.test.buckets.2.key.kw: "foo" } + - match: { aggregations.test.buckets.2.doc_count: 1 } + - match: { aggregations.test.buckets.3.key.geo: "12/2048/0" } + - match: { aggregations.test.buckets.3.key.kw: "bar" } + - match: { aggregations.test.buckets.3.doc_count: 1 } + +--- +"Simple Composite aggregation with geotile grid add aggregate after": + - skip: + version: " - 7.4.99" + reason: geotile_grid is not supported until 7.5.0 + - do: + search: + index: test + body: + aggregations: + test: + composite: + sources: [ + "geo": { + "geotile_grid": { + "field": "geo_point", + "precision": 12 + } + }, + { + "kw": { + "terms": { + "field": "keyword" + } + } + } + ] + after: { "geo": "12/730/1590", "kw": "foo" } + + - match: { hits.total.value: 6 } + - match: { hits.total.relation: "eq" } + - length: { aggregations.test.buckets: 3 } + - match: { aggregations.test.buckets.0.key.geo: "12/1236/1533" } + - match: { aggregations.test.buckets.0.key.kw: "bar" } + - match: { aggregations.test.buckets.0.doc_count: 2 } + - match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" } + - match: { aggregations.test.buckets.1.key.kw: "foo" } + - match: { aggregations.test.buckets.1.doc_count: 1 } + - match: { aggregations.test.buckets.2.key.geo: "12/2048/0" } + - match: { aggregations.test.buckets.2.key.kw: "bar" } + - match: { aggregations.test.buckets.2.doc_count: 1 } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_geohash_grid.yml b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/280_geohash_grid.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/280_geohash_grid.yml rename to modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/280_geohash_grid.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml b/modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/290_geotile_grid.yml similarity index 100% rename from rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/290_geotile_grid.yml rename to modules/geo/src/yamlRestTest/resources/rest-api-spec/test/geo_shape/290_geotile_grid.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml index 2e298441918bc..09278690f5d05 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml @@ -651,92 +651,6 @@ setup: } ] ---- -"Simple Composite aggregation with GeoTile grid": - - skip: - version: " - 7.4.99" - reason: geotile_grid is not supported until 7.5.0 - - do: - search: - rest_total_hits_as_int: true - index: test - body: - aggregations: - test: - composite: - sources: [ - "geo": { - "geotile_grid": { - "field": "geo_point", - "precision": 12 - } - }, - { - "kw": { - "terms": { - "field": "keyword" - } - } - } - ] - - - match: {hits.total: 6} - - length: { aggregations.test.buckets: 4 } - - match: { aggregations.test.buckets.0.key.geo: "12/730/1590" } - - match: { aggregations.test.buckets.0.key.kw: "foo" } - - match: { aggregations.test.buckets.0.doc_count: 1 } - - match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" } - - match: { aggregations.test.buckets.1.key.kw: "bar" } - - match: { aggregations.test.buckets.1.doc_count: 2 } - - match: { aggregations.test.buckets.2.key.geo: "12/1236/1533" } - - match: { aggregations.test.buckets.2.key.kw: "foo" } - - match: { aggregations.test.buckets.2.doc_count: 1 } - - match: { aggregations.test.buckets.3.key.geo: "12/2048/0" } - - match: { aggregations.test.buckets.3.key.kw: "bar" } - - match: { aggregations.test.buckets.3.doc_count: 1 } - ---- -"Simple Composite aggregation with geotile grid add aggregate after": - - skip: - version: " - 7.4.99" - reason: geotile_grid is not supported until 7.5.0 - - do: - search: - index: test - body: - aggregations: - test: - composite: - sources: [ - "geo": { - "geotile_grid": { - "field": "geo_point", - "precision": 12 - } - }, - { - "kw": { - "terms": { - "field": "keyword" - } - } - } - ] - after: { "geo": "12/730/1590", "kw": "foo" } - - - match: { hits.total.value: 6 } - - match: { hits.total.relation: "eq" } - - length: { aggregations.test.buckets: 3 } - - match: { aggregations.test.buckets.0.key.geo: "12/1236/1533" } - - match: { aggregations.test.buckets.0.key.kw: "bar" } - - match: { aggregations.test.buckets.0.doc_count: 2 } - - match: { aggregations.test.buckets.1.key.geo: "12/1236/1533" } - - match: { aggregations.test.buckets.1.key.kw: "foo" } - - match: { aggregations.test.buckets.1.doc_count: 1 } - - match: { aggregations.test.buckets.2.key.geo: "12/2048/0" } - - match: { aggregations.test.buckets.2.key.kw: "bar" } - - match: { aggregations.test.buckets.2.doc_count: 1 } - --- "Mixed ip and unmapped fields": - skip: diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java index 7352dc7170a21..faa6a54394b00 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/ShardReduceIT.java @@ -37,7 +37,6 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.opensearch.search.aggregations.bucket.filter.Filter; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGrid; import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.histogram.Histogram; @@ -51,8 +50,6 @@ import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram; import static org.opensearch.search.aggregations.AggregationBuilders.dateRange; import static org.opensearch.search.aggregations.AggregationBuilders.filter; -import static org.opensearch.search.aggregations.AggregationBuilders.geohashGrid; -import static org.opensearch.search.aggregations.AggregationBuilders.geotileGrid; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.search.aggregations.AggregationBuilders.histogram; import static org.opensearch.search.aggregations.AggregationBuilders.ipRange; @@ -338,36 +335,4 @@ public void testDateHistogram() throws Exception { } - public void testGeoHashGrid() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .setQuery(QueryBuilders.matchAllQuery()) - .addAggregation( - geohashGrid("grid").field("location") - .subAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).minDocCount(0)) - ) - .get(); - - assertSearchResponse(response); - - GeoGrid grid = response.getAggregations().get("grid"); - Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); - assertThat(histo.getBuckets().size(), equalTo(4)); - } - - public void testGeoTileGrid() throws Exception { - SearchResponse response = client().prepareSearch("idx") - .setQuery(QueryBuilders.matchAllQuery()) - .addAggregation( - geotileGrid("grid").field("location") - .subAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).minDocCount(0)) - ) - .get(); - - assertSearchResponse(response); - - GeoGrid grid = response.getAggregations().get("grid"); - Histogram histo = grid.getBuckets().iterator().next().getAggregations().get("histo"); - assertThat(histo.getBuckets().size(), equalTo(4)); - } - } diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java index 7cd8b3ed39051..ffc31b7cdb7c4 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/metrics/GeoCentroidIT.java @@ -35,15 +35,11 @@ import org.opensearch.action.search.SearchResponse; import org.opensearch.common.geo.GeoPoint; import org.opensearch.search.aggregations.InternalAggregation; -import org.opensearch.search.aggregations.bucket.geogrid.GeoGrid; import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.test.OpenSearchIntegTestCase; -import java.util.List; - import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.search.aggregations.AggregationBuilders.geoCentroid; -import static org.opensearch.search.aggregations.AggregationBuilders.geohashGrid; import static org.opensearch.search.aggregations.AggregationBuilders.global; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.closeTo; @@ -168,33 +164,4 @@ public void testMultiValuedField() throws Exception { assertThat(centroid.lon(), closeTo(multiCentroid.lon(), GEOHASH_TOLERANCE)); assertEquals(2 * numDocs, geoCentroid.count()); } - - public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception { - SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) - .addAggregation( - geohashGrid("geoGrid").field(SINGLE_VALUED_FIELD_NAME).subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) - ) - .get(); - assertSearchResponse(response); - - GeoGrid grid = response.getAggregations().get("geoGrid"); - assertThat(grid, notNullValue()); - assertThat(grid.getName(), equalTo("geoGrid")); - List buckets = grid.getBuckets(); - for (GeoGrid.Bucket cell : buckets) { - String geohash = cell.getKeyAsString(); - GeoPoint expectedCentroid = expectedCentroidsForGeoHash.get(geohash); - GeoCentroid centroidAgg = cell.getAggregations().get(aggName); - assertThat( - "Geohash " + geohash + " has wrong centroid latitude ", - expectedCentroid.lat(), - closeTo(centroidAgg.centroid().lat(), GEOHASH_TOLERANCE) - ); - assertThat( - "Geohash " + geohash + " has wrong centroid longitude", - expectedCentroid.lon(), - closeTo(centroidAgg.centroid().lon(), GEOHASH_TOLERANCE) - ); - } - } } diff --git a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java index a743360e1e90c..af7d4fc2e9fe5 100644 --- a/server/src/main/java/org/opensearch/plugins/SearchPlugin.java +++ b/server/src/main/java/org/opensearch/plugins/SearchPlugin.java @@ -55,6 +55,9 @@ import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.PipelineAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationParsingFunction; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.opensearch.search.aggregations.bucket.terms.SignificantTerms; import org.opensearch.search.aggregations.bucket.terms.heuristic.SignificanceHeuristic; import org.opensearch.search.aggregations.pipeline.MovAvgModel; @@ -172,6 +175,15 @@ default List> getAggregationExtentions() return emptyList(); } + /** + * Allows plugins to register new Aggregation in the {@link CompositeAggregation}. + * + * @return A {@link List} of {@link CompositeAggregationSpec} + */ + default List getCompositeAggregations() { + return emptyList(); + } + /** * The new {@link PipelineAggregator}s added by this plugin. */ @@ -532,6 +544,76 @@ public AggregationSpec setAggregatorRegistrar(Consumer aggregatorRegistrar; + private final Class valueSourceBuilderClass; + @Deprecated + /** This is added for backward compatibility, you don't need to set it, as we use aggregationType instead of + * byte code + */ + private Byte byteCode; + private final CompositeAggregationParsingFunction parsingFunction; + private final String aggregationType; + private final Writeable.Reader> reader; + + /** + * Specification for registering an aggregation in Composite Aggregation + * + * @param aggregatorRegistrar function to register the + * {@link org.opensearch.search.aggregations.support.ValuesSource} to aggregator mappings for Composite + * aggregation + * @param valueSourceBuilderClass ValueSourceBuilder class name which is building the aggregation + * @param byteCode byte code which is used in serialisation and de-serialisation to indentify which + * aggregation builder to use + * @param reader Typically, a reference to a constructor that takes a {@link StreamInput}, which is + * registered with the aggregation + * @param parsingFunction a reference function which will be used to parse the Aggregation input. + * @param aggregationType a {@link String} defined in the AggregationBuilder as type. + */ + public CompositeAggregationSpec( + final Consumer aggregatorRegistrar, + final Class> valueSourceBuilderClass, + final Byte byteCode, + final Writeable.Reader> reader, + final CompositeAggregationParsingFunction parsingFunction, + final String aggregationType + ) { + this.aggregatorRegistrar = aggregatorRegistrar; + this.valueSourceBuilderClass = valueSourceBuilderClass; + this.byteCode = byteCode; + this.parsingFunction = parsingFunction; + this.aggregationType = aggregationType; + this.reader = reader; + } + + public Consumer getAggregatorRegistrar() { + return aggregatorRegistrar; + } + + public Class getValueSourceBuilderClass() { + return valueSourceBuilderClass; + } + + public Byte getByteCode() { + return byteCode; + } + + public CompositeAggregationParsingFunction getParsingFunction() { + return parsingFunction; + } + + public String getAggregationType() { + return aggregationType; + } + + public Writeable.Reader> getReader() { + return reader; + } + } + /** * Specification for a {@link PipelineAggregator}. */ diff --git a/server/src/main/java/org/opensearch/search/DocValueFormat.java b/server/src/main/java/org/opensearch/search/DocValueFormat.java index 7e7e4f83334f5..84c46e400543a 100644 --- a/server/src/main/java/org/opensearch/search/DocValueFormat.java +++ b/server/src/main/java/org/opensearch/search/DocValueFormat.java @@ -47,7 +47,7 @@ import org.opensearch.common.time.DateUtils; import org.opensearch.geometry.utils.Geohash; import org.opensearch.index.mapper.DateFieldMapper; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import java.io.IOException; import java.math.BigInteger; diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java index 80e025a3651a8..0149f9a025bcd 100644 --- a/server/src/main/java/org/opensearch/search/SearchModule.java +++ b/server/src/main/java/org/opensearch/search/SearchModule.java @@ -126,10 +126,6 @@ import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.opensearch.search.aggregations.bucket.filter.InternalFilter; import org.opensearch.search.aggregations.bucket.filter.InternalFilters; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.global.InternalGlobal; import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -628,22 +624,6 @@ private ValuesSourceRegistry registerAggregations(List plugins) { ).addResultReader(InternalGeoDistance::new).setAggregatorRegistrar(GeoDistanceAggregationBuilder::registerAggregators), builder ); - registerAggregation( - new AggregationSpec( - GeoHashGridAggregationBuilder.NAME, - GeoHashGridAggregationBuilder::new, - GeoHashGridAggregationBuilder.PARSER - ).addResultReader(InternalGeoHashGrid::new).setAggregatorRegistrar(GeoHashGridAggregationBuilder::registerAggregators), - builder - ); - registerAggregation( - new AggregationSpec( - GeoTileGridAggregationBuilder.NAME, - GeoTileGridAggregationBuilder::new, - GeoTileGridAggregationBuilder.PARSER - ).addResultReader(InternalGeoTileGrid::new).setAggregatorRegistrar(GeoTileGridAggregationBuilder::registerAggregators), - builder - ); registerAggregation( new AggregationSpec(NestedAggregationBuilder.NAME, NestedAggregationBuilder::new, NestedAggregationBuilder::parse) .addResultReader(InternalNested::new), @@ -681,7 +661,7 @@ private ValuesSourceRegistry registerAggregations(List plugins) { registerAggregation( new AggregationSpec(CompositeAggregationBuilder.NAME, CompositeAggregationBuilder::new, CompositeAggregationBuilder.PARSER) .addResultReader(InternalComposite::new) - .setAggregatorRegistrar(CompositeAggregationBuilder::registerAggregators), + .setAggregatorRegistrar(reg -> CompositeAggregationBuilder.registerAggregators(reg, plugins)), builder ); registerAggregation( diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java b/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java index 382455093309d..9886e423bbc76 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregationBuilders.java @@ -43,10 +43,6 @@ import org.opensearch.search.aggregations.bucket.filter.Filters; import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.opensearch.search.aggregations.bucket.filter.FiltersAggregator.KeyedFilter; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.opensearch.search.aggregations.bucket.global.Global; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; @@ -261,20 +257,6 @@ public static HistogramAggregationBuilder histogram(String name) { return new HistogramAggregationBuilder(name); } - /** - * Create a new {@link InternalGeoHashGrid} aggregation with the given name. - */ - public static GeoHashGridAggregationBuilder geohashGrid(String name) { - return new GeoHashGridAggregationBuilder(name); - } - - /** - * Create a new {@link InternalGeoTileGrid} aggregation with the given name. - */ - public static GeoTileGridAggregationBuilder geotileGrid(String name) { - return new GeoTileGridAggregationBuilder(name); - } - /** * Create a new {@link SignificantTerms} aggregation with the given name. */ diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java similarity index 97% rename from server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtils.java rename to server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java index 5498b2b1a7109..6cd1823622f01 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/GeoTileUtils.java @@ -29,7 +29,7 @@ * GitHub history for details. */ -package org.opensearch.search.aggregations.bucket.geogrid; +package org.opensearch.search.aggregations.bucket; import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.util.SloppyMath; @@ -104,7 +104,7 @@ private GeoTileUtils() {} * @param parser {@link XContentParser} to parse the value from * @return int representing precision */ - static int parsePrecision(XContentParser parser) throws IOException, OpenSearchParseException { + public static int parsePrecision(XContentParser parser) throws IOException, OpenSearchParseException { final Object node = parser.currentToken().equals(XContentParser.Token.VALUE_NUMBER) ? Integer.valueOf(parser.intValue()) : parser.text(); @@ -252,7 +252,7 @@ public static String stringEncode(long hash) { /** * Decode long hash as a GeoPoint (center of the tile) */ - static GeoPoint hashToGeoPoint(long hash) { + public static GeoPoint hashToGeoPoint(long hash) { int[] res = parseHash(hash); return zxyToGeoPoint(res[0], res[1], res[2]); } @@ -260,7 +260,7 @@ static GeoPoint hashToGeoPoint(long hash) { /** * Decode a string bucket key in "zoom/x/y" format to a GeoPoint (center of the tile) */ - static GeoPoint keyToGeoPoint(String hashAsString) { + public static GeoPoint keyToGeoPoint(String hashAsString) { int[] hashAsInts = parseHash(hashAsString); return zxyToGeoPoint(hashAsInts[0], hashAsInts[1], hashAsInts[2]); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java index 8b07df3f689bf..093c2ad42722e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilder.java @@ -35,9 +35,11 @@ import org.opensearch.common.ParseField; import org.opensearch.common.io.stream.StreamInput; import org.opensearch.common.io.stream.StreamOutput; +import org.opensearch.common.io.stream.Writeable; import org.opensearch.common.xcontent.ConstructingObjectParser; import org.opensearch.common.xcontent.XContentBuilder; import org.opensearch.index.query.QueryShardContext; +import org.opensearch.plugins.SearchPlugin; import org.opensearch.search.aggregations.AbstractAggregationBuilder; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregatorFactories; @@ -47,11 +49,14 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Consumer; +import java.util.function.Function; import static org.opensearch.common.xcontent.ConstructingObjectParser.constructorArg; @@ -82,14 +87,55 @@ public class CompositeAggregationBuilder extends AbstractAggregationBuilder p.map(), AFTER_FIELD_NAME); } - public static void registerAggregators(ValuesSourceRegistry.Builder builder) { + static final Map, Byte> BUILDER_CLASS_TO_BYTE_CODE = new HashMap<>(); + static final Map BUILDER_TYPE_TO_PARSER = new HashMap<>(); + static final Map>> BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER = + new HashMap<>(); + static final Map< + String, + Writeable.Reader>> AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER = new HashMap<>(); + static final Map, String> BUILDER_CLASS_TO_AGGREGATION_TYPE = new HashMap<>(); + + public static void registerAggregators(ValuesSourceRegistry.Builder builder, final List plugins) { DateHistogramValuesSourceBuilder.register(builder); HistogramValuesSourceBuilder.register(builder); - GeoTileGridValuesSourceBuilder.register(builder); TermsValuesSourceBuilder.register(builder); + // Register All other aggregations that wants to be part of Composite Aggregation which are provided in + // Plugins along with their parsers and serialisation codes + registerCompositeAggregatorsPlugins(plugins, SearchPlugin::getCompositeAggregations, (compositeAggregationSpec) -> { + compositeAggregationSpec.getAggregatorRegistrar().accept(builder); + BUILDER_TYPE_TO_PARSER.put(compositeAggregationSpec.getAggregationType(), compositeAggregationSpec.getParsingFunction()); + // This is added for backward compatibility, so that we can move away from byte code in the serialisation + if (compositeAggregationSpec.getByteCode() != null) { + BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER.put( + (int) compositeAggregationSpec.getByteCode(), + compositeAggregationSpec.getReader() + ); + BUILDER_CLASS_TO_BYTE_CODE.put( + compositeAggregationSpec.getValueSourceBuilderClass(), + compositeAggregationSpec.getByteCode() + ); + } + AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER.put( + compositeAggregationSpec.getAggregationType(), + compositeAggregationSpec.getReader() + ); + BUILDER_CLASS_TO_AGGREGATION_TYPE.put( + compositeAggregationSpec.getValueSourceBuilderClass(), + compositeAggregationSpec.getAggregationType() + ); + }); builder.registerUsage(NAME); } + private static void registerCompositeAggregatorsPlugins( + final List plugins, + final Function> producer, + final Consumer consumer + ) { + plugins.forEach(searchPlugin -> producer.apply(searchPlugin).forEach(consumer)); + } + private List> sources; private Map after; private int size = 10; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationParsingFunction.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationParsingFunction.java new file mode 100644 index 0000000000000..344563ad20309 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationParsingFunction.java @@ -0,0 +1,22 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.bucket.composite; + +import org.opensearch.common.xcontent.XContentParser; + +import java.io.IOException; + +/** + * A functional interface which encapsulates the parsing function to be called for the aggregation which is + * also registered as CompositeAggregation. + */ +@FunctionalInterface +public interface CompositeAggregationParsingFunction { + CompositeValuesSourceBuilder parse(final String name, final XContentParser parser) throws IOException; +} diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java index 7764d367a0cec..26015ae04cf76 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceBuilder.java @@ -69,11 +69,11 @@ public abstract class CompositeValuesSourceBuilder createValuesSource( * @param missingBucket If true an explicit null bucket will represent documents with missing values. * @param hasScript true if the source contains a script that can change the value. */ - CompositeValuesSourceConfig( + public CompositeValuesSourceConfig( String name, @Nullable MappedFieldType fieldType, ValuesSource vs, @@ -113,21 +113,21 @@ SingleDimensionValuesSource createValuesSource( /** * Returns the name associated with this configuration. */ - String name() { + protected String name() { return name; } /** * Returns the {@link MappedFieldType} for this config. */ - MappedFieldType fieldType() { + public MappedFieldType fieldType() { return fieldType; } /** * Returns the {@link ValuesSource} for this configuration. */ - ValuesSource valuesSource() { + public ValuesSource valuesSource() { return vs; } @@ -135,35 +135,35 @@ ValuesSource valuesSource() { * The {@link DocValueFormat} to use for formatting the keys. * {@link DocValueFormat#RAW} means no formatting. */ - DocValueFormat format() { + public DocValueFormat format() { return format; } /** * If true, an explicit `null bucket represents documents with missing values. */ - boolean missingBucket() { + public boolean missingBucket() { return missingBucket; } /** * Return the {@link MissingOrder} for the config. */ - MissingOrder missingOrder() { + public MissingOrder missingOrder() { return missingOrder; } /** * Returns true if the source contains a script that can change the value. */ - boolean hasScript() { + protected boolean hasScript() { return hasScript; } /** * The sort order for the values source (e.g. -1 for descending and 1 for ascending). */ - int reverseMul() { + public int reverseMul() { assert reverseMul == -1 || reverseMul == 1; return reverseMul; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java index 60d7f277f7650..d8526e684f391 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesSourceParserHelper.java @@ -49,6 +49,11 @@ import java.io.IOException; import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BUILDER_CLASS_TO_AGGREGATION_TYPE; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BUILDER_CLASS_TO_BYTE_CODE; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BUILDER_TYPE_TO_PARSER; +import static org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder.BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER; /** * Helper class for obtaining values source parsers for different aggs @@ -57,7 +62,11 @@ */ public class CompositeValuesSourceParserHelper { - static , T> void declareValuesSourceFields(AbstractObjectParser objectParser) { + private static final int AGGREGATION_TYPE_REFERENCE = Byte.MAX_VALUE; + + public static , T> void declareValuesSourceFields( + AbstractObjectParser objectParser + ) { objectParser.declareField(VB::field, XContentParser::text, new ParseField("field"), ObjectParser.ValueType.STRING); objectParser.declareBoolean(VB::missingBucket, new ParseField("missing_bucket")); objectParser.declareString(VB::missingOrder, new ParseField(MissingOrder.NAME)); @@ -78,28 +87,45 @@ static , T> void declareValuesSource } public static void writeTo(CompositeValuesSourceBuilder builder, StreamOutput out) throws IOException { - final byte code; + int code = Byte.MIN_VALUE; + String aggregationType = null; if (builder.getClass() == TermsValuesSourceBuilder.class) { code = 0; } else if (builder.getClass() == DateHistogramValuesSourceBuilder.class) { code = 1; } else if (builder.getClass() == HistogramValuesSourceBuilder.class) { code = 2; - } else if (builder.getClass() == GeoTileGridValuesSourceBuilder.class) { - if (out.getVersion().before(LegacyESVersion.V_7_5_0)) { - throw new IOException( - "Attempting to serialize [" - + builder.getClass().getSimpleName() - + "] to a node with unsupported version [" - + out.getVersion() - + "]" - ); - } - code = 3; } else { - throw new IOException("invalid builder type: " + builder.getClass().getSimpleName()); + if (!BUILDER_CLASS_TO_BYTE_CODE.containsKey(builder.getClass()) + && !BUILDER_CLASS_TO_AGGREGATION_TYPE.containsKey(builder.getClass())) { + throw new IOException("invalid builder type: " + builder.getClass().getSimpleName()); + } + aggregationType = BUILDER_CLASS_TO_AGGREGATION_TYPE.get(builder.getClass()); + if (BUILDER_CLASS_TO_BYTE_CODE.containsKey(builder.getClass())) { + code = BUILDER_CLASS_TO_BYTE_CODE.get(builder.getClass()); + if (code == 3 && out.getVersion().before(LegacyESVersion.V_7_5_0)) { + throw new IOException( + "Attempting to serialize [" + + builder.getClass().getSimpleName() + + "] to a node with unsupported version [" + + out.getVersion() + + "]" + ); + } + } + } + + if (code != Byte.MIN_VALUE) { + out.writeByte((byte) code); + } else if (!BUILDER_CLASS_TO_BYTE_CODE.containsKey(builder.getClass())) { + /* + * This is added for backward compatibility when 1 data node is using the new code which is using the + * aggregation type and another is using the only byte code in the serialisation. + */ + out.writeByte((byte) AGGREGATION_TYPE_REFERENCE); + assert aggregationType != null; + out.writeString(aggregationType); } - out.writeByte(code); builder.writeTo(out); } @@ -112,10 +138,17 @@ public static CompositeValuesSourceBuilder readFrom(StreamInput in) throws IO return new DateHistogramValuesSourceBuilder(in); case 2: return new HistogramValuesSourceBuilder(in); - case 3: - return new GeoTileGridValuesSourceBuilder(in); + case AGGREGATION_TYPE_REFERENCE: + final String aggregationType = in.readString(); + if (!AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER.containsKey(aggregationType)) { + throw new IOException("Invalid aggregation type " + aggregationType); + } + return (CompositeValuesSourceBuilder) AGGREGATION_TYPE_TO_COMPOSITE_VALUE_SOURCE_READER.get(aggregationType).read(in); default: - throw new IOException("Invalid code " + code); + if (!BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER.containsKey(code)) { + throw new IOException("Invalid code " + code); + } + return (CompositeValuesSourceBuilder) BYTE_CODE_TO_COMPOSITE_VALUE_SOURCE_READER.get(code).read(in); } } @@ -143,11 +176,11 @@ public static CompositeValuesSourceBuilder fromXContent(XContentParser parser case HistogramValuesSourceBuilder.TYPE: builder = HistogramValuesSourceBuilder.parse(name, parser); break; - case GeoTileGridValuesSourceBuilder.TYPE: - builder = GeoTileGridValuesSourceBuilder.parse(name, parser); - break; default: - throw new ParsingException(parser.getTokenLocation(), "invalid source type: " + type); + if (!BUILDER_TYPE_TO_PARSER.containsKey(type)) { + throw new ParsingException(parser.getTokenLocation(), "invalid source type: " + type); + } + builder = BUILDER_TYPE_TO_PARSER.get(type).parse(name, parser); } parser.nextToken(); parser.nextToken(); @@ -163,4 +196,5 @@ public static XContentBuilder toXContent(CompositeValuesSourceBuilder source, builder.endObject(); return builder; } + } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/LongValuesSource.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/LongValuesSource.java index a7ed50507288d..ec6410c2a9377 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/LongValuesSource.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/LongValuesSource.java @@ -66,7 +66,7 @@ * * @opensearch.internal */ -class LongValuesSource extends SingleDimensionValuesSource { +public class LongValuesSource extends SingleDimensionValuesSource { private final BigArrays bigArrays; private final CheckedFunction docValuesFunc; private final LongUnaryOperator rounding; @@ -76,7 +76,7 @@ class LongValuesSource extends SingleDimensionValuesSource { private long currentValue; private boolean missingCurrentValue; - LongValuesSource( + public LongValuesSource( BigArrays bigArrays, MappedFieldType fieldType, CheckedFunction docValuesFunc, @@ -165,7 +165,7 @@ private int compareValues(long v1, long v2) { } @Override - void setAfter(Comparable value) { + protected void setAfter(Comparable value) { if (missingBucket && value == null) { afterValue = null; } else { diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java index 747a7017ec872..fe0801d6d230e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SingleDimensionValuesSource.java @@ -53,7 +53,7 @@ * * @opensearch.internal */ -abstract class SingleDimensionValuesSource> implements Releasable { +public abstract class SingleDimensionValuesSource> implements Releasable { protected final BigArrays bigArrays; protected final DocValueFormat format; @Nullable diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java b/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java index f36c4620d5b33..b4da1d10b4b68 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/AggregationInspectionHelper.java @@ -35,7 +35,6 @@ import org.opensearch.search.aggregations.bucket.composite.InternalComposite; import org.opensearch.search.aggregations.bucket.filter.InternalFilter; import org.opensearch.search.aggregations.bucket.filter.InternalFilters; -import org.opensearch.search.aggregations.bucket.geogrid.InternalGeoGrid; import org.opensearch.search.aggregations.bucket.global.InternalGlobal; import org.opensearch.search.aggregations.bucket.histogram.InternalVariableWidthHistogram; import org.opensearch.search.aggregations.bucket.histogram.InternalAutoDateHistogram; @@ -119,10 +118,6 @@ public static boolean hasValue(InternalFilter agg) { return agg.getDocCount() > 0; } - public static boolean hasValue(InternalGeoGrid agg) { - return agg.getBuckets().stream().anyMatch(bucket -> bucket.getDocCount() > 0); - } - public static boolean hasValue(InternalGlobal agg) { return agg.getDocCount() > 0; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/support/package-info.java b/server/src/main/java/org/opensearch/search/aggregations/support/package-info.java index e16e8c91b3fd0..dd2c16f1daa0e 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/support/package-info.java +++ b/server/src/main/java/org/opensearch/search/aggregations/support/package-info.java @@ -43,7 +43,7 @@ * output). A class hierarchy defines the type of values returned by the source. The top level sub-classes define type-specific behavior, * such as {@link org.opensearch.search.aggregations.support.ValuesSource.Numeric#isFloatingPoint()}. Second level subclasses are * then specialized based on where they read values from, e.g. script or field cases. There are also adapter classes like - * {@link org.opensearch.search.aggregations.bucket.geogrid.CellIdSource} which do run-time conversion from one type to another, often + * org.opensearch.search.aggregations.bucket.geogrid.CellIdSource which do run-time conversion from one type to another, often * dependent on a user specified parameter (precision in that case). *

* diff --git a/server/src/test/java/org/opensearch/search/DocValueFormatTests.java b/server/src/test/java/org/opensearch/search/DocValueFormatTests.java index 36a6eb3ae87b0..bd0fbfe69960c 100644 --- a/server/src/test/java/org/opensearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/opensearch/search/DocValueFormatTests.java @@ -48,7 +48,7 @@ import java.util.ArrayList; import java.util.List; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.longEncode; public class DocValueFormatTests extends OpenSearchTestCase { diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java index 111ce23f8a0cb..94fb6cded637d 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/AggregationsTests.java @@ -48,8 +48,6 @@ import org.opensearch.search.aggregations.bucket.composite.InternalCompositeTests; import org.opensearch.search.aggregations.bucket.filter.InternalFilterTests; import org.opensearch.search.aggregations.bucket.filter.InternalFiltersTests; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridTests; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridTests; import org.opensearch.search.aggregations.bucket.global.InternalGlobalTests; import org.opensearch.search.aggregations.bucket.histogram.InternalAutoDateHistogramTests; import org.opensearch.search.aggregations.bucket.histogram.InternalDateHistogramTests; @@ -157,8 +155,6 @@ private static List> getAggsTests() { aggsTests.add(new InternalGlobalTests()); aggsTests.add(new InternalFilterTests()); aggsTests.add(new InternalSamplerTests()); - aggsTests.add(new GeoHashGridTests()); - aggsTests.add(new GeoTileGridTests()); aggsTests.add(new InternalRangeTests()); aggsTests.add(new InternalDateRangeTests()); aggsTests.add(new InternalGeoDistanceTests()); diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilderTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilderTests.java index c4a87f3993bb4..9290183ec7312 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilderTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationBuilderTests.java @@ -32,10 +32,8 @@ package org.opensearch.search.aggregations.bucket.composite; -import org.opensearch.common.geo.GeoBoundingBoxTests; import org.opensearch.script.Script; import org.opensearch.search.aggregations.BaseAggregationTestCase; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.sort.SortOrder; @@ -74,17 +72,6 @@ private DateHistogramValuesSourceBuilder randomDateHistogramSourceBuilder() { return histo; } - private GeoTileGridValuesSourceBuilder randomGeoTileGridValuesSourceBuilder() { - GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10)); - if (randomBoolean()) { - geoTile.precision(randomIntBetween(0, GeoTileUtils.MAX_ZOOM)); - } - if (randomBoolean()) { - geoTile.geoBoundingBox(GeoBoundingBoxTests.randomBBox()); - } - return geoTile; - } - private TermsValuesSourceBuilder randomTermsSourceBuilder() { TermsValuesSourceBuilder terms = new TermsValuesSourceBuilder(randomAlphaOfLengthBetween(5, 10)); if (randomBoolean()) { @@ -118,11 +105,9 @@ private HistogramValuesSourceBuilder randomHistogramSourceBuilder() { @Override protected CompositeAggregationBuilder createTestAggregatorBuilder() { int numSources = randomIntBetween(1, 10); - numSources = 1; List> sources = new ArrayList<>(); for (int i = 0; i < numSources; i++) { - int type = randomIntBetween(0, 3); - type = 3; + int type = randomIntBetween(0, 2); switch (type) { case 0: sources.add(randomTermsSourceBuilder()); @@ -133,9 +118,6 @@ protected CompositeAggregationBuilder createTestAggregatorBuilder() { case 2: sources.add(randomHistogramSourceBuilder()); break; - case 3: - sources.add(randomGeoTileGridValuesSourceBuilder()); - break; default: throw new AssertionError("wrong branch"); } diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java index 88b2323b8adfc..25003e0b84567 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregatorTests.java @@ -32,68 +32,24 @@ package org.opensearch.search.aggregations.bucket.composite; -import org.apache.lucene.tests.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.DoublePoint; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.InetAddressPoint; -import org.apache.lucene.document.IntPoint; -import org.apache.lucene.document.LatLonPoint; import org.apache.lucene.document.LongPoint; -import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.document.SortedSetDocValuesField; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocValuesFieldExistsQuery; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.SortedNumericSortField; -import org.apache.lucene.search.SortedSetSortField; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.NumericUtils; -import org.apache.lucene.tests.util.TestUtil; import org.opensearch.OpenSearchParseException; -import org.opensearch.common.geo.GeoPoint; -import org.opensearch.common.settings.Settings; -import org.opensearch.common.text.Text; -import org.opensearch.common.time.DateFormatter; -import org.opensearch.common.time.DateFormatters; -import org.opensearch.index.Index; -import org.opensearch.index.IndexSettings; -import org.opensearch.index.mapper.DateFieldMapper; -import org.opensearch.index.mapper.DocumentMapper; -import org.opensearch.index.mapper.GeoPointFieldMapper; -import org.opensearch.index.mapper.IpFieldMapper; -import org.opensearch.index.mapper.KeywordFieldMapper; -import org.opensearch.index.mapper.MappedFieldType; -import org.opensearch.index.mapper.MapperService; -import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.search.aggregations.Aggregator; -import org.opensearch.search.aggregations.AggregatorTestCase; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.aggregations.bucket.terms.StringTerms; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; +import org.opensearch.search.aggregations.composite.BaseCompositeAggregatorTestCase; import org.opensearch.search.aggregations.metrics.InternalMax; import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder; import org.opensearch.search.aggregations.metrics.TopHits; import org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder; import org.opensearch.search.aggregations.support.ValueType; import org.opensearch.search.sort.SortOrder; -import org.opensearch.test.IndexSettingsModule; -import org.junit.After; -import org.junit.Before; import java.io.IOException; import java.net.InetAddress; @@ -109,51 +65,14 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class CompositeAggregatorTests extends AggregatorTestCase { - private static MappedFieldType[] FIELD_TYPES; - - @Override - @Before - public void setUp() throws Exception { - super.setUp(); - FIELD_TYPES = new MappedFieldType[8]; - FIELD_TYPES[0] = new KeywordFieldMapper.KeywordFieldType("keyword"); - FIELD_TYPES[1] = new NumberFieldMapper.NumberFieldType("long", NumberFieldMapper.NumberType.LONG); - FIELD_TYPES[2] = new NumberFieldMapper.NumberFieldType("double", NumberFieldMapper.NumberType.DOUBLE); - FIELD_TYPES[3] = new DateFieldMapper.DateFieldType("date", DateFormatter.forPattern("yyyy-MM-dd||epoch_millis")); - FIELD_TYPES[4] = new NumberFieldMapper.NumberFieldType("price", NumberFieldMapper.NumberType.INTEGER); - FIELD_TYPES[5] = new KeywordFieldMapper.KeywordFieldType("terms"); - FIELD_TYPES[6] = new IpFieldMapper.IpFieldType("ip"); - FIELD_TYPES[7] = new GeoPointFieldMapper.GeoPointFieldType("geo_point"); - } - - @Override - @After - public void tearDown() throws Exception { - super.tearDown(); - FIELD_TYPES = null; - } - @Override - protected MapperService mapperServiceMock() { - MapperService mapperService = mock(MapperService.class); - DocumentMapper mapper = mock(DocumentMapper.class); - when(mapper.typeText()).thenReturn(new Text("_doc")); - when(mapper.type()).thenReturn("_doc"); - when(mapperService.documentMapper()).thenReturn(mapper); - return mapperService; - } +public class CompositeAggregatorTests extends BaseCompositeAggregatorTestCase { public void testUnmappedFieldWithTerms() throws Exception { final List>> dataset = new ArrayList<>(); @@ -234,80 +153,6 @@ public void testUnmappedFieldWithTerms() throws Exception { ); } - public void testUnmappedFieldWithGeopoint() throws Exception { - final List>> dataset = new ArrayList<>(); - final String mappedFieldName = "geo_point"; - dataset.addAll( - Arrays.asList( - createDocument(mappedFieldName, new GeoPoint(48.934059, 41.610741)), - createDocument(mappedFieldName, new GeoPoint(-23.065941, 113.610741)), - createDocument(mappedFieldName, new GeoPoint(90.0, 0.0)), - createDocument(mappedFieldName, new GeoPoint(37.2343, -115.8067)), - createDocument(mappedFieldName, new GeoPoint(90.0, 0.0)) - ) - ); - - // just unmapped = no results - testSearchCase( - Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), - dataset, - () -> new CompositeAggregationBuilder("name", Arrays.asList(new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped"))), - (result) -> assertEquals(0, result.getBuckets().size()) - ); - - // unmapped missing bucket = one result - testSearchCase( - Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), - dataset, - () -> new CompositeAggregationBuilder( - "name", - Arrays.asList(new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped").missingBucket(true)) - ), - (result) -> { - assertEquals(1, result.getBuckets().size()); - assertEquals("{unmapped=null}", result.afterKey().toString()); - assertEquals("{unmapped=null}", result.getBuckets().get(0).getKeyAsString()); - assertEquals(5L, result.getBuckets().get(0).getDocCount()); - } - ); - - // field + unmapped, no missing bucket = no results - testSearchCase( - Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), - dataset, - () -> new CompositeAggregationBuilder( - "name", - Arrays.asList( - new GeoTileGridValuesSourceBuilder(mappedFieldName).field(mappedFieldName), - new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped") - ) - ), - (result) -> assertEquals(0, result.getBuckets().size()) - ); - - // field + unmapped with missing bucket = multiple results - testSearchCase( - Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery(mappedFieldName)), - dataset, - () -> new CompositeAggregationBuilder( - "name", - Arrays.asList( - new GeoTileGridValuesSourceBuilder(mappedFieldName).field(mappedFieldName), - new GeoTileGridValuesSourceBuilder("unmapped").field("unmapped").missingBucket(true) - ) - ), - (result) -> { - assertEquals(2, result.getBuckets().size()); - assertEquals("{geo_point=7/64/56, unmapped=null}", result.afterKey().toString()); - assertEquals("{geo_point=7/32/56, unmapped=null}", result.getBuckets().get(0).getKeyAsString()); - assertEquals(2L, result.getBuckets().get(0).getDocCount()); - assertEquals("{geo_point=7/64/56, unmapped=null}", result.getBuckets().get(1).getKeyAsString()); - assertEquals(3L, result.getBuckets().get(1).getDocCount()); - } - ); - - } - public void testUnmappedFieldWithHistogram() throws Exception { final List>> dataset = new ArrayList<>(); final String mappedFieldName = "price"; @@ -2483,42 +2328,6 @@ public void testWithIP() throws Exception { }); } - public void testWithGeoPoint() throws Exception { - final List>> dataset = new ArrayList<>(); - dataset.addAll( - Arrays.asList( - createDocument("geo_point", new GeoPoint(48.934059, 41.610741)), - createDocument("geo_point", new GeoPoint(-23.065941, 113.610741)), - createDocument("geo_point", new GeoPoint(90.0, 0.0)), - createDocument("geo_point", new GeoPoint(37.2343, -115.8067)), - createDocument("geo_point", new GeoPoint(90.0, 0.0)) - ) - ); - testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset, () -> { - GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point").field("geo_point"); - return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile)); - }, (result) -> { - assertEquals(2, result.getBuckets().size()); - assertEquals("{geo_point=7/64/56}", result.afterKey().toString()); - assertEquals("{geo_point=7/32/56}", result.getBuckets().get(0).getKeyAsString()); - assertEquals(2L, result.getBuckets().get(0).getDocCount()); - assertEquals("{geo_point=7/64/56}", result.getBuckets().get(1).getKeyAsString()); - assertEquals(3L, result.getBuckets().get(1).getDocCount()); - }); - - testSearchCase(Arrays.asList(new MatchAllDocsQuery(), new DocValuesFieldExistsQuery("geo_point")), dataset, () -> { - GeoTileGridValuesSourceBuilder geoTile = new GeoTileGridValuesSourceBuilder("geo_point").field("geo_point"); - return new CompositeAggregationBuilder("name", Collections.singletonList(geoTile)).aggregateAfter( - Collections.singletonMap("geo_point", "7/32/56") - ); - }, (result) -> { - assertEquals(1, result.getBuckets().size()); - assertEquals("{geo_point=7/64/56}", result.afterKey().toString()); - assertEquals("{geo_point=7/64/56}", result.getBuckets().get(0).getKeyAsString()); - assertEquals(3L, result.getBuckets().get(0).getDocCount()); - }); - } - public void testEarlyTermination() throws Exception { final List>> dataset = new ArrayList<>(); dataset.addAll( @@ -2648,193 +2457,4 @@ public void testIndexSortWithDuplicate() throws Exception { ); } } - - private void testSearchCase( - List queries, - List>> dataset, - Supplier create, - Consumer verify - ) throws IOException { - for (Query query : queries) { - executeTestCase(false, false, query, dataset, create, verify); - executeTestCase(false, true, query, dataset, create, verify); - } - } - - private void executeTestCase( - boolean forceMerge, - boolean useIndexSort, - Query query, - List>> dataset, - Supplier create, - Consumer verify - ) throws IOException { - Map types = Arrays.stream(FIELD_TYPES) - .collect(Collectors.toMap(MappedFieldType::name, Function.identity())); - CompositeAggregationBuilder aggregationBuilder = create.get(); - Sort indexSort = useIndexSort ? buildIndexSort(aggregationBuilder.sources(), types) : null; - IndexSettings indexSettings = createIndexSettings(indexSort); - try (Directory directory = newDirectory()) { - IndexWriterConfig config = newIndexWriterConfig(random(), new MockAnalyzer(random())); - if (indexSort != null) { - config.setIndexSort(indexSort); - config.setCodec(TestUtil.getDefaultCodec()); - } - try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, config)) { - Document document = new Document(); - int id = 0; - for (Map> fields : dataset) { - document.clear(); - addToDocument(id, document, fields); - indexWriter.addDocument(document); - id++; - } - if (forceMerge || rarely()) { - // forceMerge randomly or if the collector-per-leaf testing stuff would break the tests. - indexWriter.forceMerge(1); - } else { - if (dataset.size() > 0) { - int numDeletes = randomIntBetween(1, 25); - for (int i = 0; i < numDeletes; i++) { - id = randomIntBetween(0, dataset.size() - 1); - indexWriter.deleteDocuments(new Term("id", Integer.toString(id))); - document.clear(); - addToDocument(id, document, dataset.get(id)); - indexWriter.addDocument(document); - } - } - - } - } - try (IndexReader indexReader = DirectoryReader.open(directory)) { - IndexSearcher indexSearcher = new IndexSearcher(indexReader); - InternalComposite composite = searchAndReduce(indexSettings, indexSearcher, query, aggregationBuilder, FIELD_TYPES); - verify.accept(composite); - } - } - } - - private static IndexSettings createIndexSettings(Sort sort) { - Settings.Builder builder = Settings.builder(); - if (sort != null) { - String[] fields = Arrays.stream(sort.getSort()).map(SortField::getField).toArray(String[]::new); - String[] orders = Arrays.stream(sort.getSort()).map((o) -> o.getReverse() ? "desc" : "asc").toArray(String[]::new); - builder.putList("index.sort.field", fields); - builder.putList("index.sort.order", orders); - } - return IndexSettingsModule.newIndexSettings(new Index("_index", "0"), builder.build()); - } - - private void addToDocument(int id, Document doc, Map> keys) { - doc.add(new StringField("id", Integer.toString(id), Field.Store.NO)); - for (Map.Entry> entry : keys.entrySet()) { - final String name = entry.getKey(); - for (Object value : entry.getValue()) { - if (value instanceof Integer) { - doc.add(new SortedNumericDocValuesField(name, (int) value)); - doc.add(new IntPoint(name, (int) value)); - } else if (value instanceof Long) { - doc.add(new SortedNumericDocValuesField(name, (long) value)); - doc.add(new LongPoint(name, (long) value)); - } else if (value instanceof Double) { - doc.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong((double) value))); - doc.add(new DoublePoint(name, (double) value)); - } else if (value instanceof String) { - doc.add(new SortedSetDocValuesField(name, new BytesRef((String) value))); - doc.add(new StringField(name, new BytesRef((String) value), Field.Store.NO)); - } else if (value instanceof InetAddress) { - doc.add(new SortedSetDocValuesField(name, new BytesRef(InetAddressPoint.encode((InetAddress) value)))); - doc.add(new InetAddressPoint(name, (InetAddress) value)); - } else if (value instanceof GeoPoint) { - GeoPoint point = (GeoPoint) value; - doc.add( - new SortedNumericDocValuesField( - name, - GeoTileUtils.longEncode(point.lon(), point.lat(), GeoTileGridAggregationBuilder.DEFAULT_PRECISION) - ) - ); - doc.add(new LatLonPoint(name, point.lat(), point.lon())); - } else { - throw new AssertionError("invalid object: " + value.getClass().getSimpleName()); - } - } - } - } - - private static Map createAfterKey(Object... fields) { - assert fields.length % 2 == 0; - final Map map = new HashMap<>(); - for (int i = 0; i < fields.length; i += 2) { - String field = (String) fields[i]; - map.put(field, fields[i + 1]); - } - return map; - } - - @SuppressWarnings("unchecked") - private static Map> createDocument(Object... fields) { - assert fields.length % 2 == 0; - final Map> map = new HashMap<>(); - for (int i = 0; i < fields.length; i += 2) { - String field = (String) fields[i]; - if (fields[i + 1] instanceof List) { - map.put(field, (List) fields[i + 1]); - } else { - map.put(field, Collections.singletonList(fields[i + 1])); - } - } - return map; - } - - private static long asLong(String dateTime) { - return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); - } - - private static Sort buildIndexSort(List> sources, Map fieldTypes) { - List sortFields = new ArrayList<>(); - Map remainingFieldTypes = new HashMap<>(fieldTypes); - for (CompositeValuesSourceBuilder source : sources) { - MappedFieldType type = fieldTypes.remove(source.field()); - remainingFieldTypes.remove(source.field()); - SortField sortField = sortFieldFrom(type); - if (sortField == null) { - break; - } - sortFields.add(sortField); - } - while (remainingFieldTypes.size() > 0 && randomBoolean()) { - // Add extra unused sorts - List fields = new ArrayList<>(remainingFieldTypes.keySet()); - Collections.sort(fields); - String field = fields.get(between(0, fields.size() - 1)); - SortField sortField = sortFieldFrom(remainingFieldTypes.remove(field)); - if (sortField != null) { - sortFields.add(sortField); - } - } - return sortFields.size() > 0 ? new Sort(sortFields.toArray(new SortField[0])) : null; - } - - private static SortField sortFieldFrom(MappedFieldType type) { - if (type instanceof KeywordFieldMapper.KeywordFieldType) { - return new SortedSetSortField(type.name(), false); - } else if (type instanceof DateFieldMapper.DateFieldType) { - return new SortedNumericSortField(type.name(), SortField.Type.LONG, false); - } else if (type instanceof NumberFieldMapper.NumberFieldType) { - switch (type.typeName()) { - case "byte": - case "short": - case "integer": - return new SortedNumericSortField(type.name(), SortField.Type.INT, false); - case "long": - return new SortedNumericSortField(type.name(), SortField.Type.LONG, false); - case "float": - case "double": - return new SortedNumericSortField(type.name(), SortField.Type.DOUBLE, false); - default: - return null; - } - } - return null; - } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index dfe4034650594..1443208a1d2fc 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -34,14 +34,15 @@ import org.opensearch.common.geo.GeoPoint; import org.opensearch.geometry.Rectangle; +import org.opensearch.search.aggregations.bucket.GeoTileUtils; import org.opensearch.test.OpenSearchTestCase; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.checkPrecisionRange; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.hashToGeoPoint; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.keyToGeoPoint; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; -import static org.opensearch.search.aggregations.bucket.geogrid.GeoTileUtils.stringEncode; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.MAX_ZOOM; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.checkPrecisionRange; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.hashToGeoPoint; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.keyToGeoPoint; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.longEncode; +import static org.opensearch.search.aggregations.bucket.GeoTileUtils.stringEncode; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java new file mode 100644 index 0000000000000..7d00772913d6e --- /dev/null +++ b/test/framework/src/main/java/org/opensearch/search/aggregations/composite/BaseCompositeAggregatorTestCase.java @@ -0,0 +1,310 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.composite; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSortField; +import org.apache.lucene.search.SortedSetSortField; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.TestUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; +import org.junit.After; +import org.junit.Before; +import org.opensearch.common.settings.Settings; +import org.opensearch.common.text.Text; +import org.opensearch.common.time.DateFormatter; +import org.opensearch.common.time.DateFormatters; +import org.opensearch.index.Index; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.DocumentMapper; +import org.opensearch.index.mapper.IpFieldMapper; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.search.aggregations.AggregatorTestCase; +import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; +import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.composite.InternalComposite; +import org.opensearch.test.IndexSettingsModule; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Base class for the Aggregator Tests which are registered under Composite Aggregation. + */ +public class BaseCompositeAggregatorTestCase extends AggregatorTestCase { + + protected static List FIELD_TYPES; + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + FIELD_TYPES = new ArrayList<>(); + FIELD_TYPES.add(new KeywordFieldMapper.KeywordFieldType("keyword")); + FIELD_TYPES.add(new NumberFieldMapper.NumberFieldType("long", NumberFieldMapper.NumberType.LONG)); + FIELD_TYPES.add(new NumberFieldMapper.NumberFieldType("double", NumberFieldMapper.NumberType.DOUBLE)); + FIELD_TYPES.add(new DateFieldMapper.DateFieldType("date", DateFormatter.forPattern("yyyy-MM-dd||epoch_millis"))); + FIELD_TYPES.add(new NumberFieldMapper.NumberFieldType("price", NumberFieldMapper.NumberType.INTEGER)); + FIELD_TYPES.add(new KeywordFieldMapper.KeywordFieldType("terms")); + FIELD_TYPES.add(new IpFieldMapper.IpFieldType("ip")); + } + + @Override + @After + public void tearDown() throws Exception { + super.tearDown(); + FIELD_TYPES = null; + } + + @Override + protected MapperService mapperServiceMock() { + MapperService mapperService = mock(MapperService.class); + DocumentMapper mapper = mock(DocumentMapper.class); + when(mapper.typeText()).thenReturn(new Text("_doc")); + when(mapper.type()).thenReturn("_doc"); + when(mapperService.documentMapper()).thenReturn(mapper); + return mapperService; + } + + protected static Map> createDocument(Object... fields) { + assert fields.length % 2 == 0; + final Map> map = new HashMap<>(); + for (int i = 0; i < fields.length; i += 2) { + String field = (String) fields[i]; + if (fields[i + 1] instanceof List) { + map.put(field, (List) fields[i + 1]); + } else { + map.put(field, Collections.singletonList(fields[i + 1])); + } + } + return map; + } + + protected void testSearchCase( + List queries, + List>> dataset, + Supplier create, + Consumer verify + ) throws IOException { + for (Query query : queries) { + executeTestCase(false, false, query, dataset, create, verify); + executeTestCase(false, true, query, dataset, create, verify); + } + } + + protected void executeTestCase( + boolean forceMerge, + boolean useIndexSort, + Query query, + List>> dataset, + Supplier create, + Consumer verify + ) throws IOException { + Map types = FIELD_TYPES.stream().collect(Collectors.toMap(MappedFieldType::name, Function.identity())); + CompositeAggregationBuilder aggregationBuilder = create.get(); + Sort indexSort = useIndexSort ? buildIndexSort(aggregationBuilder.sources(), types) : null; + IndexSettings indexSettings = createIndexSettings(indexSort); + try (Directory directory = newDirectory()) { + IndexWriterConfig config = newIndexWriterConfig(random(), new MockAnalyzer(random())); + if (indexSort != null) { + config.setIndexSort(indexSort); + config.setCodec(TestUtil.getDefaultCodec()); + } + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory, config)) { + Document document = new Document(); + int id = 0; + for (Map> fields : dataset) { + document.clear(); + addToDocument(id, document, fields); + indexWriter.addDocument(document); + id++; + } + if (forceMerge || rarely()) { + // forceMerge randomly or if the collector-per-leaf testing stuff would break the tests. + indexWriter.forceMerge(1); + } else { + if (dataset.size() > 0) { + int numDeletes = randomIntBetween(1, 25); + for (int i = 0; i < numDeletes; i++) { + id = randomIntBetween(0, dataset.size() - 1); + indexWriter.deleteDocuments(new Term("id", Integer.toString(id))); + document.clear(); + addToDocument(id, document, dataset.get(id)); + indexWriter.addDocument(document); + } + } + + } + } + try (IndexReader indexReader = DirectoryReader.open(directory)) { + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + InternalComposite composite = searchAndReduce( + indexSettings, + indexSearcher, + query, + aggregationBuilder, + FIELD_TYPES.toArray(new MappedFieldType[0]) + ); + verify.accept(composite); + } + } + } + + protected void addToDocument(int id, Document doc, Map> keys) { + doc.add(new StringField("id", Integer.toString(id), Field.Store.NO)); + for (Map.Entry> entry : keys.entrySet()) { + final String name = entry.getKey(); + for (Object value : entry.getValue()) { + if (value instanceof Integer) { + doc.add(new SortedNumericDocValuesField(name, (int) value)); + doc.add(new IntPoint(name, (int) value)); + } else if (value instanceof Long) { + doc.add(new SortedNumericDocValuesField(name, (long) value)); + doc.add(new LongPoint(name, (long) value)); + } else if (value instanceof Double) { + doc.add(new SortedNumericDocValuesField(name, NumericUtils.doubleToSortableLong((double) value))); + doc.add(new DoublePoint(name, (double) value)); + } else if (value instanceof String) { + doc.add(new SortedSetDocValuesField(name, new BytesRef((String) value))); + doc.add(new StringField(name, new BytesRef((String) value), Field.Store.NO)); + } else if (value instanceof InetAddress) { + doc.add(new SortedSetDocValuesField(name, new BytesRef(InetAddressPoint.encode((InetAddress) value)))); + doc.add(new InetAddressPoint(name, (InetAddress) value)); + } else { + if (!addValueToDocument(doc, name, value)) throw new AssertionError( + "invalid object: " + value.getClass().getSimpleName() + ); + } + } + } + } + + /** + * Override this function to handle any specific type of value you want to add in the document for doing the + * composite aggregation. If you have added another Composite Aggregation Type then you must override this + * function so that your field value can be added in the document correctly. + * + * @param doc {@link Document} + * @param name {@link String} Field Name + * @param value {@link Object} Field value + * @return boolean true or false, based on if value is added or not + */ + protected boolean addValueToDocument(final Document doc, final String name, final Object value) { + return false; + } + + protected static Sort buildIndexSort(List> sources, Map fieldTypes) { + List sortFields = new ArrayList<>(); + Map remainingFieldTypes = new HashMap<>(fieldTypes); + for (CompositeValuesSourceBuilder source : sources) { + MappedFieldType type = fieldTypes.remove(source.field()); + remainingFieldTypes.remove(source.field()); + SortField sortField = sortFieldFrom(type); + if (sortField == null) { + break; + } + sortFields.add(sortField); + } + while (remainingFieldTypes.size() > 0 && randomBoolean()) { + // Add extra unused sorts + List fields = new ArrayList<>(remainingFieldTypes.keySet()); + Collections.sort(fields); + String field = fields.get(between(0, fields.size() - 1)); + SortField sortField = sortFieldFrom(remainingFieldTypes.remove(field)); + if (sortField != null) { + sortFields.add(sortField); + } + } + return sortFields.size() > 0 ? new Sort(sortFields.toArray(new SortField[0])) : null; + } + + protected static SortField sortFieldFrom(MappedFieldType type) { + if (type instanceof KeywordFieldMapper.KeywordFieldType) { + return new SortedSetSortField(type.name(), false); + } else if (type instanceof DateFieldMapper.DateFieldType) { + return new SortedNumericSortField(type.name(), SortField.Type.LONG, false); + } else if (type instanceof NumberFieldMapper.NumberFieldType) { + switch (type.typeName()) { + case "byte": + case "short": + case "integer": + return new SortedNumericSortField(type.name(), SortField.Type.INT, false); + case "long": + return new SortedNumericSortField(type.name(), SortField.Type.LONG, false); + case "float": + case "double": + return new SortedNumericSortField(type.name(), SortField.Type.DOUBLE, false); + default: + return null; + } + } + return null; + } + + protected static IndexSettings createIndexSettings(Sort sort) { + Settings.Builder builder = Settings.builder(); + if (sort != null) { + String[] fields = Arrays.stream(sort.getSort()).map(SortField::getField).toArray(String[]::new); + String[] orders = Arrays.stream(sort.getSort()).map((o) -> o.getReverse() ? "desc" : "asc").toArray(String[]::new); + builder.putList("index.sort.field", fields); + builder.putList("index.sort.order", orders); + } + return IndexSettingsModule.newIndexSettings(new Index("_index", "0"), builder.build()); + } + + protected static Map createAfterKey(Object... fields) { + assert fields.length % 2 == 0; + final Map map = new HashMap<>(); + for (int i = 0; i < fields.length; i += 2) { + String field = (String) fields[i]; + map.put(field, fields[i + 1]); + } + return map; + } + + protected static long asLong(String dateTime) { + return DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(dateTime)).toInstant().toEpochMilli(); + } +} diff --git a/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java b/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java index a4099d66de28e..5325c48e16913 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalAggregationTestCase.java @@ -68,10 +68,6 @@ import org.opensearch.search.aggregations.bucket.filter.FiltersAggregationBuilder; import org.opensearch.search.aggregations.bucket.filter.ParsedFilter; import org.opensearch.search.aggregations.bucket.filter.ParsedFilters; -import org.opensearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoHashGrid; -import org.opensearch.search.aggregations.bucket.geogrid.ParsedGeoTileGrid; import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.opensearch.search.aggregations.bucket.global.ParsedGlobal; import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; @@ -275,8 +271,6 @@ public ReduceContext forFinalReduction() { map.put(GlobalAggregationBuilder.NAME, (p, c) -> ParsedGlobal.fromXContent(p, (String) c)); map.put(FilterAggregationBuilder.NAME, (p, c) -> ParsedFilter.fromXContent(p, (String) c)); map.put(InternalSampler.PARSER_NAME, (p, c) -> ParsedSampler.fromXContent(p, (String) c)); - map.put(GeoHashGridAggregationBuilder.NAME, (p, c) -> ParsedGeoHashGrid.fromXContent(p, (String) c)); - map.put(GeoTileGridAggregationBuilder.NAME, (p, c) -> ParsedGeoTileGrid.fromXContent(p, (String) c)); map.put(RangeAggregationBuilder.NAME, (p, c) -> ParsedRange.fromXContent(p, (String) c)); map.put(DateRangeAggregationBuilder.NAME, (p, c) -> ParsedDateRange.fromXContent(p, (String) c)); map.put(GeoDistanceAggregationBuilder.NAME, (p, c) -> ParsedGeoDistance.fromXContent(p, (String) c));